Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
#include <gtest/gtest.h>
#include <string>
#include <vector>
#include <fstream>
#include <algorithm>
#include <collection_manager.h>
#include "collection.h"
class CollectionOverrideTest : public ::testing::Test {
protected:
Store *store;
CollectionManager & collectionManager = CollectionManager::get_instance();
std::atomic<bool> quit = false;
Collection *coll_mul_fields;
void setupCollection() {
std::string state_dir_path = "/tmp/typesense_test/collection_override";
LOG(INFO) << "Truncating and creating: " << state_dir_path;
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
collectionManager.init(store, 1.0, "auth_key", quit);
collectionManager.load(8, 1000);
std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");
std::vector<field> fields = {
field("title", field_types::STRING, false),
field("starring", field_types::STRING, true),
field("cast", field_types::STRING_ARRAY, true),
field("points", field_types::INT32, false)
};
coll_mul_fields = collectionManager.get_collection("coll_mul_fields").get();
if(coll_mul_fields == nullptr) {
coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get();
}
std::string json_line;
while (std::getline(infile, json_line)) {
coll_mul_fields->add(json_line);
}
infile.close();
}
virtual void SetUp() {
setupCollection();
}
virtual void TearDown() {
collectionManager.drop_collection("coll_mul_fields");
collectionManager.dispose();
delete store;
}
};
TEST_F(CollectionOverrideTest, ExcludeIncludeExactQueryMatch) {
Config::get_instance().set_enable_search_analytics(true);
nlohmann::json override_json = {
{"id", "exclude-rule"},
{
"rule", {
{"query", "of"},
{"match", override_t::MATCH_EXACT}
}
}
};
override_json["excludes"] = nlohmann::json::array();
override_json["excludes"][0] = nlohmann::json::object();
override_json["excludes"][0]["id"] = "4";
override_json["excludes"][1] = nlohmann::json::object();
override_json["excludes"][1]["id"] = "11";
override_t override;
override_t::parse(override_json, "", override);
coll_mul_fields->add_override(override);
std::vector<std::string> facets = {"cast"};
Option<nlohmann::json> res_op = coll_mul_fields->search("of", {"title"}, "", facets, {}, {0}, 10);
ASSERT_TRUE(res_op.ok());
nlohmann::json results = res_op.get();
ASSERT_EQ(3, results["hits"].size());
ASSERT_EQ(3, results["found"].get<uint32_t>());
ASSERT_EQ(6, results["facet_counts"][0]["counts"].size());
ASSERT_STREQ("12", results["hits"][0]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("5", results["hits"][1]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("17", results["hits"][2]["document"]["id"].get<std::string>().c_str());
// include
nlohmann::json override_json_include = {
{"id", "include-rule"},
{
"rule", {
{"query", "in"},
{"match", override_t::MATCH_EXACT}
}
}
};
override_json_include["includes"] = nlohmann::json::array();
override_json_include["includes"][0] = nlohmann::json::object();
override_json_include["includes"][0]["id"] = "0";
override_json_include["includes"][0]["position"] = 1;
override_json_include["includes"][1] = nlohmann::json::object();
override_json_include["includes"][1]["id"] = "3";
override_json_include["includes"][1]["position"] = 2;
override_t override_include;
override_t::parse(override_json_include, "", override_include);
coll_mul_fields->add_override(override_include);
res_op = coll_mul_fields->search("in", {"title"}, "", {}, {}, {0}, 10);
ASSERT_TRUE(res_op.ok());
results = res_op.get();
ASSERT_EQ(3, results["hits"].size());
ASSERT_EQ(3, results["found"].get<uint32_t>());
ASSERT_FALSE(results.contains("metadata"));
ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("3", results["hits"][1]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("13", results["hits"][2]["document"]["id"].get<std::string>().c_str());
// curated results should be marked as such
ASSERT_EQ(true, results["hits"][0]["curated"].get<bool>());
ASSERT_EQ(true, results["hits"][1]["curated"].get<bool>());
ASSERT_EQ(0, results["hits"][2].count("curated"));
coll_mul_fields->remove_override("exclude-rule");
coll_mul_fields->remove_override("include-rule");
// contains cases
nlohmann::json override_contains_inc = {
{"id", "include-rule"},
{
"rule", {
{"query", "will"},
{"match", override_t::MATCH_CONTAINS}
}
}
};
override_contains_inc["includes"] = nlohmann::json::array();
override_contains_inc["includes"][0] = nlohmann::json::object();
override_contains_inc["includes"][0]["id"] = "0";
override_contains_inc["includes"][0]["position"] = 1;
override_contains_inc["includes"][1] = nlohmann::json::object();
override_contains_inc["includes"][1]["id"] = "1";
override_contains_inc["includes"][1]["position"] = 7; // purposely setting it way out
override_t override_inc_contains;
override_t::parse(override_contains_inc, "", override_inc_contains);
coll_mul_fields->add_override(override_inc_contains);
res_op = coll_mul_fields->search("will smith", {"title"}, "", {}, {}, {0}, 10);
ASSERT_TRUE(res_op.ok());
results = res_op.get();
ASSERT_EQ(4, results["hits"].size());
ASSERT_EQ(4, results["found"].get<uint32_t>());
ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("3", results["hits"][1]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("2", results["hits"][2]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("1", results["hits"][3]["document"]["id"].get<std::string>().c_str());
// partial word should not match
res_op = coll_mul_fields->search("dowillow", {"title"}, "", {}, {}, {0}, 10);
ASSERT_TRUE(res_op.ok());
results = res_op.get();
ASSERT_EQ(0, results["hits"].size());
ASSERT_EQ(0, results["found"].get<uint32_t>());
// ability to disable overrides
bool enable_overrides = false;
res_op = coll_mul_fields->search("will", {"title"}, "", {}, {}, {0}, 10,
1, FREQUENCY, {false}, 0, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 0, {}, {}, {}, 0,
"<mark>", "</mark>", {1}, 10000, true, false, enable_overrides);
ASSERT_TRUE(res_op.ok());
results = res_op.get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ(2, results["found"].get<uint32_t>());
ASSERT_STREQ("3", results["hits"][0]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("2", results["hits"][1]["document"]["id"].get<std::string>().c_str());
enable_overrides = true;
res_op = coll_mul_fields->search("will", {"title"}, "", {}, {}, {0}, 10,
1, FREQUENCY, {false}, 0, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 0, {}, {}, {}, 0,
"<mark>", "</mark>", {1}, 10000, true, false, enable_overrides);
ASSERT_TRUE(res_op.ok());
results = res_op.get();
ASSERT_EQ(4, results["hits"].size());
ASSERT_EQ(4, results["found"].get<uint32_t>());
coll_mul_fields->remove_override("include-rule");
Config::get_instance().set_enable_search_analytics(false);
}
TEST_F(CollectionOverrideTest, OverrideJSONValidation) {
nlohmann::json exclude_json = {
{"id", "exclude-rule"},
{
"rule", {
{"query", "of"},
{"match", override_t::MATCH_EXACT}
}
}
};
exclude_json["excludes"] = nlohmann::json::array();
exclude_json["excludes"][0] = nlohmann::json::object();
exclude_json["excludes"][0]["id"] = 11;
override_t override1;
auto parse_op = override_t::parse(exclude_json, "", override1);
ASSERT_FALSE(parse_op.ok());
ASSERT_STREQ("Exclusion `id` must be a string.", parse_op.error().c_str());
nlohmann::json include_json = {
{"id", "include-rule"},
{
"rule", {
{"query", "of"},
{"match", override_t::MATCH_EXACT}
}
}
};
include_json["includes"] = nlohmann::json::array();
include_json["includes"][0] = nlohmann::json::object();
include_json["includes"][0]["id"] = "11";
override_t override2;
parse_op = override_t::parse(include_json, "", override2);
ASSERT_FALSE(parse_op.ok());
ASSERT_STREQ("Inclusion definition must define both `id` and `position` keys.", parse_op.error().c_str());
include_json["includes"][0]["position"] = "1";
parse_op = override_t::parse(include_json, "", override2);
ASSERT_FALSE(parse_op.ok());
ASSERT_STREQ("Inclusion `position` must be an integer.", parse_op.error().c_str());
include_json["includes"][0]["position"] = 1;
parse_op = override_t::parse(include_json, "", override2);
ASSERT_TRUE(parse_op.ok());
nlohmann::json include_json2 = {
{"id", "include-rule"},
{
"rule", {
{"query", "of"},
{"match", override_t::MATCH_EXACT}
}
}
};
parse_op = override_t::parse(include_json2, "", override2);
ASSERT_FALSE(parse_op.ok());
ASSERT_STREQ("Must contain one of: `includes`, `excludes`, `metadata`, `filter_by`, `sort_by`, "
"`remove_matched_tokens`, `replace_query`.", parse_op.error().c_str());
include_json2["includes"] = nlohmann::json::array();
include_json2["includes"][0] = 100;
parse_op = override_t::parse(include_json2, "", override2);
ASSERT_FALSE(parse_op.ok());
ASSERT_STREQ("The `includes` value must be an array of objects.", parse_op.error().c_str());
nlohmann::json exclude_json2 = {
{"id", "exclude-rule"},
{
"rule", {
{"query", "of"},
{"match", override_t::MATCH_EXACT}
}
}
};
exclude_json2["excludes"] = nlohmann::json::array();
exclude_json2["excludes"][0] = "100";
parse_op = override_t::parse(exclude_json2, "", override2);
ASSERT_FALSE(parse_op.ok());
ASSERT_STREQ("The `excludes` value must be an array of objects.", parse_op.error().c_str());
}
TEST_F(CollectionOverrideTest, IncludeHitsFilterOverrides) {
// Check facet field highlight for overridden results
nlohmann::json override_json_include = {
{"id", "include-rule"},
{
"rule", {
{"query", "not-found"},
{"match", override_t::MATCH_EXACT}
}
},
{"metadata", {{"foo", "bar"}}},
};
override_json_include["includes"] = nlohmann::json::array();
override_json_include["includes"][0] = nlohmann::json::object();
override_json_include["includes"][0]["id"] = "0";
override_json_include["includes"][0]["position"] = 1;
override_json_include["includes"][1] = nlohmann::json::object();
override_json_include["includes"][1]["id"] = "2";
override_json_include["includes"][1]["position"] = 2;
override_json_include["filter_curated_hits"] = true;
override_t override_include;
override_t::parse(override_json_include, "", override_include);
coll_mul_fields->add_override(override_include);
std::map<std::string, override_t*> overrides = coll_mul_fields->get_overrides().get();
ASSERT_EQ(1, overrides.size());
auto override_json = overrides.at("include-rule")->to_json();
ASSERT_TRUE(override_json.contains("filter_curated_hits"));
ASSERT_TRUE(override_json["filter_curated_hits"].get<bool>());
auto results = coll_mul_fields->search("not-found", {"title"}, "points:>70", {"starring"}, {}, {0}, 10, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "starring: will").get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("bar", results["metadata"]["foo"].get<std::string>());
// disable filter curation option
override_json_include["filter_curated_hits"] = false;
override_t::parse(override_json_include, "", override_include);
coll_mul_fields->add_override(override_include);
results = coll_mul_fields->search("not-found", {"title"}, "points:>70", {"starring"}, {}, {0}, 10, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "starring: will").get();
ASSERT_EQ(2, results["hits"].size());
// remove filter curation option: by default no filtering should be done
override_json_include.erase("filter_curated_hits");
override_t::parse(override_json_include, "", override_include);
coll_mul_fields->add_override(override_include);
results = coll_mul_fields->search("not-found", {"title"}, "points:>70", {"starring"}, {}, {0}, 10, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "starring: will").get();
ASSERT_EQ(2, results["hits"].size());
// query param configuration should take precedence over override level config
results = coll_mul_fields->search("not-found", {"title"}, "points:>70", {"starring"}, {}, {0}, 10, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "",
30, 5,
"", 10, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {off}, 32767, 32767, 2, 1).get();
ASSERT_EQ(1, results["hits"].size());
// try disabling and overriding
override_json_include["filter_curated_hits"] = false;
override_t::parse(override_json_include, "", override_include);
coll_mul_fields->add_override(override_include);
results = coll_mul_fields->search("not-found", {"title"}, "points:>70", {"starring"}, {}, {0}, 10, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "",
30, 5,
"", 10, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {off}, 32767, 32767, 2, 1).get();
ASSERT_EQ(1, results["hits"].size());
// try enabling and overriding
override_json_include["filter_curated_hits"] = true;
override_t::parse(override_json_include, "", override_include);
coll_mul_fields->add_override(override_include);
results = coll_mul_fields->search("not-found", {"title"}, "points:>70", {"starring"}, {}, {0}, 10, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "",
30, 5,
"", 10, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {off}, 32767, 32767, 2, 0).get();
ASSERT_EQ(1, results["hits"].size());
}
TEST_F(CollectionOverrideTest, ExcludeIncludeFacetFilterQuery) {
// Check facet field highlight for overridden results
nlohmann::json override_json_include = {
{"id", "include-rule"},
{
"rule", {
{"query", "not-found"},
{"match", override_t::MATCH_EXACT}
}
}
};
override_json_include["includes"] = nlohmann::json::array();
override_json_include["includes"][0] = nlohmann::json::object();
override_json_include["includes"][0]["id"] = "0";
override_json_include["includes"][0]["position"] = 1;
override_json_include["includes"][1] = nlohmann::json::object();
override_json_include["includes"][1]["id"] = "2";
override_json_include["includes"][1]["position"] = 2;
override_t override_include;
override_t::parse(override_json_include, "", override_include);
coll_mul_fields->add_override(override_include);
std::map<std::string, override_t*> overrides = coll_mul_fields->get_overrides().get();
ASSERT_EQ(1, overrides.size());
auto override_json = overrides.at("include-rule")->to_json();
ASSERT_FALSE(override_json.contains("filter_by"));
ASSERT_TRUE(override_json.contains("remove_matched_tokens"));
ASSERT_TRUE(override_json.contains("filter_curated_hits"));
ASSERT_FALSE(override_json["remove_matched_tokens"].get<bool>());
ASSERT_FALSE(override_json["filter_curated_hits"].get<bool>());
auto results = coll_mul_fields->search("not-found", {"title"}, "", {"starring"}, {}, {0}, 10, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "starring: will").get();
ASSERT_EQ("<mark>Will</mark> Ferrell", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>());
ASSERT_EQ("Will Ferrell", results["facet_counts"][0]["counts"][0]["value"].get<std::string>());
ASSERT_EQ(1, results["facet_counts"][0]["counts"][0]["count"].get<size_t>());
coll_mul_fields->remove_override("include-rule");
// facet count is okay when results are excluded
nlohmann::json override_json_exclude = {
{"id", "exclude-rule"},
{
"rule", {
{"query", "the"},
{"match", override_t::MATCH_EXACT}
}
}
};
override_json_exclude["excludes"] = nlohmann::json::array();
override_json_exclude["excludes"][0] = nlohmann::json::object();
override_json_exclude["excludes"][0]["id"] = "10";
override_t override;
override_t::parse(override_json_exclude, "", override);
coll_mul_fields->add_override(override);
results = coll_mul_fields->search("the", {"title"}, "", {"starring"}, {}, {0}, 10, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "starring: scott").get();
ASSERT_EQ(9, results["found"].get<size_t>());
// "count" would be `2` without exclusion
ASSERT_EQ("<mark>Scott</mark> Glenn", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>());
ASSERT_EQ(1, results["facet_counts"][0]["counts"][0]["count"].get<size_t>());
ASSERT_EQ("Kristin <mark>Scott</mark> Thomas", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>());
ASSERT_EQ(1, results["facet_counts"][0]["counts"][1]["count"].get<size_t>());
// ensure per_page is respected
// first with per_page = 0
results = coll_mul_fields->search("the", {"title"}, "", {"starring"}, {}, {0}, 0, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "starring: scott").get();
ASSERT_EQ(9, results["found"].get<size_t>());
ASSERT_EQ(0, results["hits"].size());
coll_mul_fields->remove_override("exclude-rule");
// now with per_page = 1, and an include query
coll_mul_fields->add_override(override_include);
results = coll_mul_fields->search("not-found", {"title"}, "", {"starring"}, {}, {0}, 1, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "").get();
ASSERT_EQ(2, results["found"].get<size_t>());
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
// should be able to replace existing override
override_include.rule.query = "found";
coll_mul_fields->add_override(override_include);
ASSERT_STREQ("found", coll_mul_fields->get_overrides().get()["include-rule"]->rule.query.c_str());
coll_mul_fields->remove_override("include-rule");
}
TEST_F(CollectionOverrideTest, FilterCuratedHitsSlideToCoverMissingSlots) {
// when some of the curated hits are filtered away, lower ranked hits must be pulled up
nlohmann::json override_json_include = {
{"id", "include-rule"},
{
"rule", {
{"query", "scott"},
{"match", override_t::MATCH_EXACT}
}
}
};
// first 2 hits won't match the filter, 3rd position should float up to position 1
override_json_include["includes"] = nlohmann::json::array();
override_json_include["includes"][0] = nlohmann::json::object();
override_json_include["includes"][0]["id"] = "7";
override_json_include["includes"][0]["position"] = 1;
override_json_include["includes"][1] = nlohmann::json::object();
override_json_include["includes"][1]["id"] = "17";
override_json_include["includes"][1]["position"] = 2;
override_json_include["includes"][2] = nlohmann::json::object();
override_json_include["includes"][2]["id"] = "10";
override_json_include["includes"][2]["position"] = 3;
override_json_include["filter_curated_hits"] = true;
override_t override_include;
override_t::parse(override_json_include, "", override_include);
coll_mul_fields->add_override(override_include);
auto results = coll_mul_fields->search("scott", {"starring"}, "points:>55", {}, {}, {0}, 10, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "").get();
ASSERT_EQ(3, results["hits"].size());
ASSERT_EQ("10", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("11", results["hits"][1]["document"]["id"].get<std::string>());
ASSERT_EQ("12", results["hits"][2]["document"]["id"].get<std::string>());
// another curation where there is an ID missing in the middle
override_json_include = {
{"id", "include-rule"},
{
"rule", {
{"query", "glenn"},
{"match", override_t::MATCH_EXACT}
}
}
};
// middle hit ("10") will not satisfy filter, so "11" will move to position 2
override_json_include["includes"] = nlohmann::json::array();
override_json_include["includes"][0] = nlohmann::json::object();
override_json_include["includes"][0]["id"] = "9";
override_json_include["includes"][0]["position"] = 1;
override_json_include["includes"][1] = nlohmann::json::object();
override_json_include["includes"][1]["id"] = "10";
override_json_include["includes"][1]["position"] = 2;
override_json_include["includes"][2] = nlohmann::json::object();
override_json_include["includes"][2]["id"] = "11";
override_json_include["includes"][2]["position"] = 3;
override_json_include["filter_curated_hits"] = true;
override_t override_include2;
override_t::parse(override_json_include, "", override_include2);
coll_mul_fields->add_override(override_include2);
results = coll_mul_fields->search("glenn", {"starring"}, "points:[43,86]", {}, {}, {0}, 10, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "").get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("9", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("11", results["hits"][1]["document"]["id"].get<std::string>());
}
TEST_F(CollectionOverrideTest, SimpleOverrideStopProcessing) {
Collection *coll1;
std::vector<field> fields = {field("name", field_types::STRING, false),
field("price", field_types::FLOAT, true),
field("points", field_types::INT32, false)};
coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "Amazing Shoes";
doc1["price"] = 399.99;
doc1["points"] = 30;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["name"] = "Fast Joggers";
doc2["price"] = 49.99;
doc2["points"] = 5;
nlohmann::json doc3;
doc3["id"] = "2";
doc3["name"] = "Comfortable Sneakers";
doc3["price"] = 19.99;
doc3["points"] = 1;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
ASSERT_TRUE(coll1->add(doc3.dump()).ok());
std::vector<sort_by> sort_fields = { sort_by("_text_match", "DESC"), sort_by("points", "DESC") };
nlohmann::json override_json_include = {
{"id", "include-rule-1"},
{
"rule", {
{"query", "shoes"},
{"match", override_t::MATCH_EXACT}
}
},
{"stop_processing", false}
};
// first 2 hits won't match the filter, 3rd position should float up to position 1
override_json_include["includes"] = nlohmann::json::array();
override_json_include["includes"][0] = nlohmann::json::object();
override_json_include["includes"][0]["id"] = "2";
override_json_include["includes"][0]["position"] = 1;
override_t override_include1;
auto op = override_t::parse(override_json_include, "include-rule-1", override_include1);
ASSERT_TRUE(op.ok());
coll1->add_override(override_include1);
override_json_include["id"] = "include-rule-2";
override_json_include["includes"] = nlohmann::json::array();
override_json_include["includes"][0] = nlohmann::json::object();
override_json_include["includes"][0]["id"] = "1";
override_json_include["includes"][0]["position"] = 2;
override_t override_include2;
op = override_t::parse(override_json_include, "include-rule-2", override_include2);
ASSERT_TRUE(op.ok());
coll1->add_override(override_include2);
auto results = coll1->search("shoes", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
ASSERT_EQ(3, results["hits"].size());
ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
ASSERT_EQ("0", results["hits"][2]["document"]["id"].get<std::string>());
// now with stop processing enabled for the first rule
override_include1.stop_processing = true;
coll1->add_override(override_include1);
results = coll1->search("shoes", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
// check that default value for stop_processing is true
nlohmann::json override_json_test = {
{"id", "include-rule-test"},
{
"rule", {
{"query", "fast"},
{"match", override_t::MATCH_CONTAINS}
}
},
};
override_json_test["includes"] = nlohmann::json::array();
override_json_test["includes"][0] = nlohmann::json::object();
override_json_test["includes"][0]["id"] = "2";
override_json_test["includes"][0]["position"] = 1;
override_t override_include_test;
op = override_t::parse(override_json_test, "include-rule-test", override_include_test);
ASSERT_TRUE(op.ok());
ASSERT_TRUE(override_include_test.stop_processing);
}
TEST_F(CollectionOverrideTest, IncludeOverrideWithFilterBy) {
Collection *coll1;
std::vector<field> fields = {field("name", field_types::STRING, false),
field("price", field_types::FLOAT, true),
field("points", field_types::INT32, false)};
coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "Amazing Shoes";
doc1["price"] = 399.99;
doc1["points"] = 30;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["name"] = "Fast Shoes";
doc2["price"] = 49.99;
doc2["points"] = 5;
nlohmann::json doc3;
doc3["id"] = "2";
doc3["name"] = "Comfortable Shoes";
doc3["price"] = 199.99;
doc3["points"] = 1;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
ASSERT_TRUE(coll1->add(doc3.dump()).ok());
std::vector<sort_by> sort_fields = { sort_by("_text_match", "DESC"), sort_by("points", "DESC") };
nlohmann::json override_json_include = {
{"id", "include-rule-1"},
{
"rule", {
{"query", "shoes"},
{"match", override_t::MATCH_EXACT}
}
},
{"filter_curated_hits", false},
{"stop_processing", false},
{"remove_matched_tokens", false},
{"filter_by", "price: >55"}
};
override_json_include["includes"] = nlohmann::json::array();
override_json_include["includes"][0] = nlohmann::json::object();
override_json_include["includes"][0]["id"] = "2";
override_json_include["includes"][0]["position"] = 1;
override_t override_include1;
auto op = override_t::parse(override_json_include, "include-rule-1", override_include1);
ASSERT_TRUE(op.ok());
coll1->add_override(override_include1);
auto results = coll1->search("shoes", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
// when filter by does not match any result, curated result should still show up
// because `filter_curated_hits` is false
results = coll1->search("shoes", {"name"}, "points:1000",
{}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
// when bad filter by clause is used in override
override_json_include = {
{"id", "include-rule-2"},
{
"rule", {
{"query", "test"},
{"match", override_t::MATCH_EXACT}
}
},
{"filter_curated_hits", false},
{"stop_processing", false},
{"remove_matched_tokens", false},
{"filter_by", "price >55"}
};
override_json_include["includes"] = nlohmann::json::array();
override_json_include["includes"][0] = nlohmann::json::object();
override_json_include["includes"][0]["id"] = "2";
override_json_include["includes"][0]["position"] = 1;
override_t override_include2;
op = override_t::parse(override_json_include, "include-rule-2", override_include2);
ASSERT_TRUE(op.ok());
coll1->add_override(override_include2);
results = coll1->search("random-name", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
ASSERT_EQ(0, results["hits"].size());
}
TEST_F(CollectionOverrideTest, ReplaceQuery) {
Collection *coll1;
std::vector<field> fields = {field("name", field_types::STRING, false),
field("points", field_types::INT32, false)};
coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "Amazing Shoes";
doc1["points"] = 30;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["name"] = "Fast Shoes";
doc2["points"] = 50;
nlohmann::json doc3;
doc3["id"] = "2";
doc3["name"] = "Comfortable Socks";
doc3["points"] = 1;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
ASSERT_TRUE(coll1->add(doc3.dump()).ok());
std::vector<sort_by> sort_fields = { sort_by("_text_match", "DESC"), sort_by("points", "DESC") };
nlohmann::json override_json = R"({
"id": "rule-1",
"rule": {
"query": "boots",
"match": "exact"
},
"replace_query": "shoes"
})"_json;
override_t override_rule;
auto op = override_t::parse(override_json, "rule-1", override_rule);
ASSERT_TRUE(op.ok());
coll1->add_override(override_rule);
auto results = coll1->search("boots", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
// don't allow both remove_matched_tokens and replace_query
override_json["remove_matched_tokens"] = true;
op = override_t::parse(override_json, "rule-1", override_rule);
ASSERT_FALSE(op.ok());
ASSERT_EQ("Only one of `replace_query` or `remove_matched_tokens` can be specified.", op.error());
// it's okay when it's explicitly set to false
override_json["remove_matched_tokens"] = false;
op = override_t::parse(override_json, "rule-1", override_rule);
ASSERT_TRUE(op.ok());
}
TEST_F(CollectionOverrideTest, RuleQueryMustBeCaseInsensitive) {
Collection *coll1;
std::vector<field> fields = {field("name", field_types::STRING, false),
field("points", field_types::INT32, false)};
coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "Amazing Shoes";
doc1["points"] = 30;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["name"] = "Tennis Ball";
doc2["points"] = 50;
nlohmann::json doc3;
doc3["id"] = "2";
doc3["name"] = "Golf Ball";
doc3["points"] = 1;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
ASSERT_TRUE(coll1->add(doc3.dump()).ok());
std::vector<sort_by> sort_fields = { sort_by("_text_match", "DESC"), sort_by("points", "DESC") };
nlohmann::json override_json = R"({
"id": "rule-1",
"rule": {
"query": "GrEat",
"match": "contains"
},
"replace_query": "amazing"
})"_json;
override_t override_rule;
auto op = override_t::parse(override_json, "rule-1", override_rule);
ASSERT_TRUE(op.ok());
coll1->add_override(override_rule);
override_json = R"({
"id": "rule-2",
"rule": {
"query": "BaLL",
"match": "contains"
},
"filter_by": "points: 1"
})"_json;
override_t override_rule2;
op = override_t::parse(override_json, "rule-2", override_rule2);
ASSERT_TRUE(op.ok());
coll1->add_override(override_rule2);
auto results = coll1->search("great shoes", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
results = coll1->search("ball", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
}
TEST_F(CollectionOverrideTest, RuleQueryWithAccentedChars) {
Collection *coll1;
std::vector<field> fields = {field("name", field_types::STRING, false),
field("color", field_types::STRING, false),
field("points", field_types::INT32, false)};
coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "Green";
doc1["color"] = "Green";
doc1["points"] = 30;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
std::vector<sort_by> sort_fields = { sort_by("_text_match", "DESC"), sort_by("points", "DESC") };
nlohmann::json override_json = R"({
"id": "rule-1",
"rule": {
"query": "Grün",
"match": "contains"
},
"filter_by":"color:green",
"filter_curated_hits":true
})"_json;
override_t override_rule;
auto op = override_t::parse(override_json, "rule-1", override_rule);
ASSERT_TRUE(op.ok());
coll1->add_override(override_rule);
auto results = coll1->search("grün", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
}
TEST_F(CollectionOverrideTest, WindowForRule) {
Collection *coll1;
std::vector<field> fields = {field("name", field_types::STRING, false),
field("points", field_types::INT32, false)};
coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "Amazing Shoes";
doc1["points"] = 30;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
std::vector<sort_by> sort_fields = { sort_by("_text_match", "DESC"), sort_by("points", "DESC") };
nlohmann::json override_json = R"({
"id": "rule-1",
"rule": {
"query": "boots",
"match": "exact"
},
"replace_query": "shoes"
})"_json;
override_t override_rule;
auto op = override_t::parse(override_json, "rule-1", override_rule);
ASSERT_TRUE(op.ok());
coll1->add_override(override_rule);
auto results = coll1->search("boots", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
// rule must not match when window_start is set into the future
override_json["effective_from_ts"] = 35677971263; // year 3100, here we come! ;)
op = override_t::parse(override_json, "rule-1", override_rule);
ASSERT_TRUE(op.ok());
coll1->add_override(override_rule);
results = coll1->search("boots", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
ASSERT_EQ(0, results["hits"].size());
// rule must not match when window_end is set into the past
override_json["effective_from_ts"] = -1;
override_json["effective_to_ts"] = 965388863;
op = override_t::parse(override_json, "rule-1", override_rule);
ASSERT_TRUE(op.ok());
coll1->add_override(override_rule);
results = coll1->search("boots", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
ASSERT_EQ(0, results["hits"].size());
// resetting both should bring the override back in action
override_json["effective_from_ts"] = 965388863;
override_json["effective_to_ts"] = 35677971263;
op = override_t::parse(override_json, "rule-1", override_rule);
ASSERT_TRUE(op.ok());
coll1->add_override(override_rule);
results = coll1->search("boots", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
ASSERT_EQ(1, results["hits"].size());
}
TEST_F(CollectionOverrideTest, FilterRule) {
Collection *coll1;
std::vector<field> fields = {field("name", field_types::STRING, false),
field("points", field_types::INT32, false)};
coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "Amazing Shoes";
doc1["points"] = 30;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["name"] = "Fast Shoes";
doc2["points"] = 50;
nlohmann::json doc3;
doc3["id"] = "2";
doc3["name"] = "Comfortable Socks";
doc3["points"] = 1;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
ASSERT_TRUE(coll1->add(doc3.dump()).ok());
std::vector<sort_by> sort_fields = { sort_by("_text_match", "DESC"), sort_by("points", "DESC") };
nlohmann::json override_json = R"({
"id": "rule-1",
"rule": {
"query": "*",
"match": "exact",
"filter_by": "points: 50"
},
"includes": [{
"id": "0",
"position": 1
}]
})"_json;
override_t override_rule;
auto op = override_t::parse(override_json, "rule-1", override_rule);
ASSERT_TRUE(op.ok());
coll1->add_override(override_rule);
auto results = coll1->search("*", {}, "points: 50",
{}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
// empty query should not trigger override even though it will be deemed as wildcard search
results = coll1->search("", {"name"}, "points: 50",
{}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
// check to_json
nlohmann::json override_json_ser = override_rule.to_json();
ASSERT_EQ("points: 50", override_json_ser["rule"]["filter_by"]);
// without q/match
override_json = R"({
"id": "rule-2",
"rule": {
"filter_by": "points: 1"
},
"includes": [{
"id": "0",
"position": 1
}]
})"_json;
override_t override_rule2;
op = override_t::parse(override_json, "rule-2", override_rule2);
ASSERT_TRUE(op.ok());
coll1->add_override(override_rule2);
results = coll1->search("socks", {"name"}, "points: 1",
{}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("2", results["hits"][1]["document"]["id"].get<std::string>());
override_json_ser = override_rule2.to_json();
ASSERT_EQ("points: 1", override_json_ser["rule"]["filter_by"]);
ASSERT_EQ(0, override_json_ser["rule"].count("query"));
ASSERT_EQ(0, override_json_ser["rule"].count("match"));
}
TEST_F(CollectionOverrideTest, CurationGroupingNonCuratedHitsShouldNotAppearOutside) {
Collection *coll1;
std::vector<field> fields = {field("title", field_types::STRING, false),
field("group_id", field_types::STRING, true),};
coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 3, fields).get();
}
nlohmann::json doc;
doc["id"] = "1";
doc["title"] = "The Harry Potter 1";
doc["group_id"] = "hp";
ASSERT_TRUE(coll1->add(doc.dump()).ok());
doc["id"] = "2";
doc["title"] = "The Harry Potter 2";
doc["group_id"] = "hp";
ASSERT_TRUE(coll1->add(doc.dump()).ok());
doc["id"] = "3";
doc["title"] = "Lord of the Rings";
doc["group_id"] = "lotr";
ASSERT_TRUE(coll1->add(doc.dump()).ok());
nlohmann::json override_json = R"({
"id": "rule-1",
"rule": {
"query": "*",
"match": "exact"
},
"includes": [{
"id": "2",
"position": 1
}]
})"_json;
override_t override_rule;
auto op = override_t::parse(override_json, "rule-1", override_rule);
ASSERT_TRUE(op.ok());
coll1->add_override(override_rule);
override_json = R"({
"id": "rule-2",
"rule": {
"query": "the",
"match": "exact"
},
"includes": [{
"id": "2",
"position": 1
}]
})"_json;
override_t override_rule2;
op = override_t::parse(override_json, "rule-2", override_rule2);
ASSERT_TRUE(op.ok());
coll1->add_override(override_rule2);
auto results = coll1->search("*", {"title"}, "", {}, {}, {0}, 50, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
"", 10,
"", {}, {"group_id"}, 2).get();
// when only one of the 2 records belonging to a record is used for curation, the other record
// should not appear back
ASSERT_EQ(2, results["found"].get<size_t>());
ASSERT_EQ(1, results["grouped_hits"][0]["hits"].size());
ASSERT_EQ(1, results["grouped_hits"][1]["hits"].size());
ASSERT_EQ("2", results["grouped_hits"][0]["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("3", results["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>());
// same for keyword search
results = coll1->search("the", {"title"}, "", {}, {}, {0}, 50, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
"", 10,
"", {}, {"group_id"}, 2).get();
// when only one of the 2 records belonging to a record is used for curation, the other record
// should not appear back
ASSERT_EQ(2, results["found"].get<size_t>());
ASSERT_EQ(1, results["grouped_hits"][0]["hits"].size());
ASSERT_EQ(1, results["grouped_hits"][1]["hits"].size());
ASSERT_EQ("2", results["grouped_hits"][0]["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("3", results["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>());
}
TEST_F(CollectionOverrideTest, PinnedAndHiddenHits) {
auto pinned_hits = "13:1,4:2";
// basic pinning
auto results = coll_mul_fields->search("the", {"title"}, "", {"starring"}, {}, {0}, 50, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "starring: will", 30, 5,
"", 10,
pinned_hits, {}).get();
ASSERT_EQ(10, results["found"].get<size_t>());
ASSERT_STREQ("13", results["hits"][0]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("4", results["hits"][1]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("11", results["hits"][2]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("16", results["hits"][3]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("6", results["hits"][4]["document"]["id"].get<std::string>().c_str());
// pinning + filtering
results = coll_mul_fields->search("of", {"title"}, "points:>58", {}, {}, {0}, 50, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
"", 10,
pinned_hits, {}).get();
ASSERT_EQ(5, results["found"].get<size_t>());
ASSERT_STREQ("13", results["hits"][0]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("4", results["hits"][1]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("11", results["hits"][2]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("12", results["hits"][3]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("5", results["hits"][4]["document"]["id"].get<std::string>().c_str());
// pinning + filtering with filter_curated_hits: true
pinned_hits = "14:1,4:2";
results = coll_mul_fields->search("of", {"title"}, "points:>58", {}, {}, {0}, 50, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
"", 10, pinned_hits, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {off}, 32767, 32767, 2, 1).get();
ASSERT_EQ(4, results["found"].get<size_t>());
ASSERT_STREQ("14", results["hits"][0]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("11", results["hits"][1]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("12", results["hits"][2]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("5", results["hits"][3]["document"]["id"].get<std::string>().c_str());
ASSERT_EQ("The Silence <mark>of</mark> the Lambs", results["hits"][1]["highlights"][0]["snippet"].get<std::string>());
ASSERT_EQ("Confessions <mark>of</mark> a Shopaholic", results["hits"][2]["highlights"][0]["snippet"].get<std::string>());
ASSERT_EQ("Percy Jackson: Sea <mark>of</mark> Monsters", results["hits"][3]["highlights"][0]["snippet"].get<std::string>());
// both pinning and hiding
pinned_hits = "13:1,4:2";
std::string hidden_hits="11,16";
results = coll_mul_fields->search("the", {"title"}, "", {"starring"}, {}, {0}, 50, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "starring: will", 30, 5,
"", 10,
pinned_hits, hidden_hits).get();
ASSERT_STREQ("13", results["hits"][0]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("4", results["hits"][1]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("6", results["hits"][2]["document"]["id"].get<std::string>().c_str());
// paginating such that pinned hits appear on second page
pinned_hits = "13:4,4:5";
results = coll_mul_fields->search("the", {"title"}, "", {"starring"}, {}, {0}, 2, 2, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "starring: will", 30, 5,
"", 10,
pinned_hits, hidden_hits).get();
ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("13", results["hits"][1]["document"]["id"].get<std::string>().c_str());
// take precedence over override rules
nlohmann::json override_json_include = {
{"id", "include-rule"},
{
"rule", {
{"query", "the"},
{"match", override_t::MATCH_EXACT}
}
}
};
// trying to include an ID that is also being hidden via `hidden_hits` query param will not work
// as pinned and hidden hits will take precedence over override rules
override_json_include["includes"] = nlohmann::json::array();
override_json_include["includes"][0] = nlohmann::json::object();
override_json_include["includes"][0]["id"] = "11";
override_json_include["includes"][0]["position"] = 2;
override_json_include["includes"][1] = nlohmann::json::object();
override_json_include["includes"][1]["id"] = "8";
override_json_include["includes"][1]["position"] = 1;
override_t override_include;
override_t::parse(override_json_include, "", override_include);
coll_mul_fields->add_override(override_include);
results = coll_mul_fields->search("the", {"title"}, "", {"starring"}, {}, {0}, 50, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "starring: will", 30, 5,
"", 10,
{}, {hidden_hits}).get();
ASSERT_EQ(8, results["found"].get<size_t>());
ASSERT_STREQ("8", results["hits"][0]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("6", results["hits"][1]["document"]["id"].get<std::string>().c_str());
}
TEST_F(CollectionOverrideTest, PinnedHitsSmallerThanPageSize) {
auto pinned_hits = "17:1,13:4,11:3";
// pinned hits larger than page size: check that pagination works
// without overrides:
// 11, 16, 6, 8, 1, 0, 10, 4, 13, 17
auto results = coll_mul_fields->search("the", {"title"}, "", {"starring"}, {}, {0}, 8, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "starring: will", 30, 5,
"", 10,
pinned_hits, {}).get();
std::vector<size_t> expected_ids_p1 = {17, 16, 11, 13, 6, 8, 1, 0};
ASSERT_EQ(10, results["found"].get<size_t>());
ASSERT_EQ(8, results["hits"].size());
for(size_t i=0; i<8; i++) {
ASSERT_EQ(expected_ids_p1[i], std::stoi(results["hits"][i]["document"]["id"].get<std::string>()));
}
std::vector<size_t> expected_ids_p2 = {10, 4};
results = coll_mul_fields->search("the", {"title"}, "", {"starring"}, {}, {0}, 8, 2, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "starring: will", 30, 5,
"", 10,
pinned_hits, {}).get();
ASSERT_EQ(10, results["found"].get<size_t>());
ASSERT_EQ(2, results["hits"].size());
for(size_t i=0; i<2; i++) {
ASSERT_EQ(expected_ids_p2[i], std::stoi(results["hits"][i]["document"]["id"].get<std::string>()));
}
}
TEST_F(CollectionOverrideTest, PinnedHitsLargerThanPageSize) {
auto pinned_hits = "6:1,1:2,16:3,11:4";
// pinned hits larger than page size: check that pagination works
auto results = coll_mul_fields->search("the", {"title"}, "", {"starring"}, {}, {0}, 2, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "starring: will", 30, 5,
"", 10,
pinned_hits, {}).get();
ASSERT_EQ(10, results["found"].get<size_t>());
ASSERT_EQ(2, results["hits"].size());
ASSERT_STREQ("6", results["hits"][0]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("1", results["hits"][1]["document"]["id"].get<std::string>().c_str());
results = coll_mul_fields->search("the", {"title"}, "", {"starring"}, {}, {0}, 2, 2, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "starring: will", 30, 5,
"", 10,
pinned_hits, {}).get();
ASSERT_EQ(10, results["found"].get<size_t>());
ASSERT_EQ(2, results["hits"].size());
ASSERT_STREQ("16", results["hits"][0]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("11", results["hits"][1]["document"]["id"].get<std::string>().c_str());
results = coll_mul_fields->search("the", {"title"}, "", {"starring"}, {}, {0}, 2, 3, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "starring: will", 30, 5,
"", 10,
pinned_hits, {}).get();
ASSERT_EQ(10, results["found"].get<size_t>());
ASSERT_EQ(2, results["hits"].size());
ASSERT_STREQ("8", results["hits"][0]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("0", results["hits"][1]["document"]["id"].get<std::string>().c_str());
}
TEST_F(CollectionOverrideTest, PinnedHitsWhenThereAreNotEnoughResults) {
auto pinned_hits = "6:1,1:2,11:5";
// multiple pinned hits specified, but query produces no result
auto results = coll_mul_fields->search("not-foundquery", {"title"}, "", {"starring"}, {}, {0}, 10, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "starring: will", 30, 5,
"", 10,
pinned_hits, {}).get();
ASSERT_EQ(3, results["found"].get<size_t>());
ASSERT_EQ(3, results["hits"].size());
ASSERT_STREQ("6", results["hits"][0]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("1", results["hits"][1]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("11", results["hits"][2]["document"]["id"].get<std::string>().c_str());
// multiple pinned hits but only single result
results = coll_mul_fields->search("burgundy", {"title"}, "", {"starring"}, {}, {0}, 10, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "starring: will", 30, 5,
"", 10,
pinned_hits, {}).get();
ASSERT_EQ(4, results["found"].get<size_t>());
ASSERT_EQ(4, results["hits"].size());
ASSERT_STREQ("6", results["hits"][0]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("1", results["hits"][1]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("0", results["hits"][2]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("11", results["hits"][3]["document"]["id"].get<std::string>().c_str());
}
TEST_F(CollectionOverrideTest, HiddenHitsHidingSingleResult) {
Collection *coll1;
std::vector<field> fields = {field("title", field_types::STRING, false),
field("points", field_types::INT32, false),};
coll1 = collectionManager.get_collection("coll1").get();
if (coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
std::vector<std::vector<std::string>> records = {
{"Down There by the Train"}
};
for (size_t i = 0; i < records.size(); i++) {
nlohmann::json doc;
doc["id"] = std::to_string(i);
doc["title"] = records[i][0];
doc["points"] = i;
ASSERT_TRUE(coll1->add(doc.dump()).ok());
}
std::string hidden_hits="0";
auto results = coll1->search("the train", {"title"}, "", {}, {}, {0}, 50, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
"", 10,
"", hidden_hits).get();
ASSERT_EQ(0, results["found"].get<size_t>());
ASSERT_EQ(0, results["hits"].size());
results = coll1->search("the train", {"title"}, "points:0", {}, {}, {0}, 50, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
"", 10,
"", hidden_hits).get();
ASSERT_EQ(0, results["found"].get<size_t>());
ASSERT_EQ(0, results["hits"].size());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionOverrideTest, PinnedHitsGrouping) {
auto pinned_hits = "6:1,8:1,1:2,13:3,4:3";
// without any grouping parameter, only the first ID in a position should be picked
// and other IDs should appear in their original positions
auto results = coll_mul_fields->search("the", {"title"}, "", {"starring"}, {}, {0}, 50, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "starring: will", 30, 5,
"", 10,
pinned_hits, {}).get();
ASSERT_EQ(10, results["found"].get<size_t>());
ASSERT_STREQ("6", results["hits"][0]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("1", results["hits"][1]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("13", results["hits"][2]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("11", results["hits"][3]["document"]["id"].get<std::string>().c_str());
// pinned hits should be marked as curated
ASSERT_EQ(true, results["hits"][0]["curated"].get<bool>());
ASSERT_EQ(true, results["hits"][1]["curated"].get<bool>());
ASSERT_EQ(true, results["hits"][2]["curated"].get<bool>());
ASSERT_EQ(0, results["hits"][3].count("curated"));
// with grouping
results = coll_mul_fields->search("the", {"title"}, "", {"starring"}, {}, {0}, 50, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "starring: will", 30, 5,
"", 10,
pinned_hits, {}, {"cast"}, 2).get();
ASSERT_EQ(8, results["found"].get<size_t>());
ASSERT_EQ(1, results["grouped_hits"][0]["group_key"].size());
ASSERT_EQ(2, results["grouped_hits"][0]["group_key"][0].size());
ASSERT_STREQ("Chris Evans", results["grouped_hits"][0]["group_key"][0][0].get<std::string>().c_str());
ASSERT_STREQ("Scarlett Johansson", results["grouped_hits"][0]["group_key"][0][1].get<std::string>().c_str());
ASSERT_STREQ("6", results["grouped_hits"][0]["hits"][0]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("8", results["grouped_hits"][0]["hits"][1]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("1", results["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("13", results["grouped_hits"][2]["hits"][0]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("4", results["grouped_hits"][2]["hits"][1]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("11", results["grouped_hits"][3]["hits"][0]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("16", results["grouped_hits"][4]["hits"][0]["document"]["id"].get<std::string>().c_str());
}
TEST_F(CollectionOverrideTest, PinnedHitsGroupingNonPinnedHitsShouldNotAppearOutside) {
Collection *coll1;
std::vector<field> fields = {field("title", field_types::STRING, false),
field("group_id", field_types::STRING, true),};
coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 3, fields).get();
}
nlohmann::json doc;
doc["id"] = "1";
doc["title"] = "The Harry Potter 1";
doc["group_id"] = "hp";
ASSERT_TRUE(coll1->add(doc.dump()).ok());
doc["id"] = "2";
doc["title"] = "The Harry Potter 2";
doc["group_id"] = "hp";
ASSERT_TRUE(coll1->add(doc.dump()).ok());
doc["id"] = "3";
doc["title"] = "Lord of the Rings";
doc["group_id"] = "lotr";
ASSERT_TRUE(coll1->add(doc.dump()).ok());
auto pinned_hits = "2:1";
auto results = coll1->search("*", {"title"}, "", {}, {}, {0}, 50, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
"", 10,
pinned_hits, {}, {"group_id"}, 2).get();
// when only one of the 2 records belonging to a record is used for curation, the other record
// should not appear back
ASSERT_EQ(2, results["found"].get<size_t>());
ASSERT_EQ(1, results["grouped_hits"][0]["hits"].size());
ASSERT_EQ(1, results["grouped_hits"][1]["hits"].size());
ASSERT_EQ("2", results["grouped_hits"][0]["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("3", results["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>());
// same for keyword search
results = coll1->search("the", {"title"}, "", {}, {}, {0}, 50, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
"", 10,
pinned_hits, {}, {"group_id"}, 2).get();
// when only one of the 2 records belonging to a record is used for curation, the other record
// should not appear back
ASSERT_EQ(2, results["found"].get<size_t>());
ASSERT_EQ(1, results["grouped_hits"][0]["hits"].size());
ASSERT_EQ(1, results["grouped_hits"][1]["hits"].size());
ASSERT_EQ("2", results["grouped_hits"][0]["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("3", results["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>());
}
TEST_F(CollectionOverrideTest, PinnedHitsWithWildCardQuery) {
Collection *coll1;
std::vector<field> fields = {field("title", field_types::STRING, false),
field("points", field_types::INT32, false),};
coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 3, fields, "points").get();
}
size_t num_indexed = 0;
for(size_t i=0; i<311; i++) {
nlohmann::json doc;
doc["id"] = std::to_string(i);
doc["title"] = "Title " + std::to_string(i);
doc["points"] = i;
ASSERT_TRUE(coll1->add(doc.dump()).ok());
num_indexed++;
}
auto pinned_hits = "7:1,4:2";
auto results = coll1->search("*", {"title"}, "", {}, {}, {0}, 30, 11, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
"", 10,
pinned_hits, {}, {}, {0}, "", "", {}).get();
ASSERT_EQ(311, results["found"].get<size_t>());
ASSERT_EQ(11, results["hits"].size());
std::vector<size_t> expected_ids = {12, 11, 10, 9, 8, 6, 5, 3, 2, 1, 0}; // 4 and 7 should be missing
for(size_t i=0; i<11; i++) {
ASSERT_EQ(expected_ids[i], std::stoi(results["hits"][i]["document"]["id"].get<std::string>()));
}
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionOverrideTest, HiddenHitsWithWildCardQuery) {
Collection *coll1;
std::vector<field> fields = {field("title", field_types::STRING, false),
field("points", field_types::INT32, false),};
coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 3, fields, "points").get();
}
for(size_t i=0; i<5; i++) {
nlohmann::json doc;
doc["id"] = std::to_string(i);
doc["title"] = "Title " + std::to_string(i);
doc["points"] = i;
ASSERT_TRUE(coll1->add(doc.dump()).ok());
}
auto hidden_hits = "1";
auto results = coll1->search("*", {"title"}, "", {}, {}, {0}, 30, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
"", 10,
{}, hidden_hits, {}, {0}, "", "", {}).get();
ASSERT_EQ(4, results["found"].get<size_t>());
ASSERT_EQ(4, results["hits"].size());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionOverrideTest, PinnedHitsIdsHavingColon) {
Collection *coll1;
std::vector<field> fields = {field("url", field_types::STRING, true),
field("points", field_types::INT32, false)};
std::vector<sort_by> sort_fields = { sort_by("points", "DESC") };
coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
}
for(size_t i=1; i<=10; i++) {
nlohmann::json doc;
doc["id"] = std::string("https://example.com/") + std::to_string(i);
doc["url"] = std::string("https://example.com/") + std::to_string(i);
doc["points"] = i;
coll1->add(doc.dump());
}
std::vector<std::string> query_fields = {"url"};
std::vector<std::string> facets;
std::string pinned_hits_str = "https://example.com/1:1, https://example.com/3:2"; // can have space
auto res_op = coll1->search("*", {"url"}, "", {}, {}, {0}, 25, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
"", 10,
pinned_hits_str, {});
ASSERT_TRUE(res_op.ok());
auto res = res_op.get();
ASSERT_EQ(10, res["found"].get<size_t>());
ASSERT_STREQ("https://example.com/1", res["hits"][0]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("https://example.com/3", res["hits"][1]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("https://example.com/10", res["hits"][2]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("https://example.com/9", res["hits"][3]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("https://example.com/2", res["hits"][9]["document"]["id"].get<std::string>().c_str());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionOverrideTest, DynamicFilteringExactMatchBasics) {
Collection *coll1;
std::vector<field> fields = {field("name", field_types::STRING, false),
field("category", field_types::STRING, true),
field("brand", field_types::STRING, true),
field("points", field_types::INT32, false)};
coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "Amazing Shoes";
doc1["category"] = "shoes";
doc1["brand"] = "Nike";
doc1["points"] = 3;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["name"] = "Track Gym";
doc2["category"] = "shoes";
doc2["brand"] = "Adidas";
doc2["points"] = 5;
nlohmann::json doc3;
doc3["id"] = "2";
doc3["name"] = "Running Shoes";
doc3["category"] = "sports";
doc3["brand"] = "Nike";
doc3["points"] = 5;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
ASSERT_TRUE(coll1->add(doc3.dump()).ok());
std::vector<sort_by> sort_fields = { sort_by("_text_match", "DESC"), sort_by("points", "DESC") };
auto results = coll1->search("shoes", {"name", "category", "brand"}, "",
{}, sort_fields, {2, 2, 2}, 10).get();
ASSERT_EQ(3, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
ASSERT_EQ("2", results["hits"][2]["document"]["id"].get<std::string>());
// with override, results will be different
nlohmann::json override_json = {
{"id", "dynamic-cat-filter"},
{
"rule", {
{"query", "{category}"},
{"match", override_t::MATCH_EXACT}
}
},
{"remove_matched_tokens", true},
{"filter_by", "category: {category}"}
};
override_t override;
auto op = override_t::parse(override_json, "dynamic-cat-filter", override);
ASSERT_TRUE(op.ok());
coll1->add_override(override);
override_json = {
{"id", "dynamic-brand-cat-filter"},
{
"rule", {
{"query", "{brand} {category}"},
{"match", override_t::MATCH_EXACT}
}
},
{"remove_matched_tokens", true},
{"filter_by", "category: {category} && brand: {brand}"}
};
op = override_t::parse(override_json, "dynamic-brand-cat-filter", override);
ASSERT_TRUE(op.ok());
coll1->add_override(override);
override_json = {
{"id", "dynamic-brand-filter"},
{
"rule", {
{"query", "{brand}"},
{"match", override_t::MATCH_EXACT}
}
},
{"remove_matched_tokens", true},
{"filter_by", "brand: {brand}"}
};
override_json["includes"] = nlohmann::json::array();
override_json["includes"][0] = nlohmann::json::object();
override_json["includes"][0]["id"] = "0";
override_json["includes"][0]["position"] = 1;
op = override_t::parse(override_json, "dynamic-brand-filter", override);
ASSERT_TRUE(op.ok());
coll1->add_override(override);
results = coll1->search("shoes", {"name", "category", "brand"}, "",
{}, sort_fields, {2, 2, 2}, 10).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
ASSERT_EQ(0, results["hits"][0]["highlights"].size());
ASSERT_EQ(0, results["hits"][1]["highlights"].size());
// should not apply filter for non-exact case
results = coll1->search("running shoes", {"name", "category", "brand"}, "",
{}, sort_fields, {2, 2, 2}, 10, 1, FREQUENCY, {false}, 10).get();
ASSERT_EQ(3, results["hits"].size());
results = coll1->search("adidas shoes", {"name", "category", "brand"}, "",
{}, sort_fields, {2, 2, 2}, 10, 1, FREQUENCY, {false}, 10).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
// dynamic brand filter + explicit ID include
results = coll1->search("adidas", {"name", "category", "brand"}, "",
{}, sort_fields, {2, 2, 2}, 10).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
// with bad override
nlohmann::json override_json_bad1 = {
{"id", "dynamic-filters-bad1"},
{
"rule", {
{"query", "{brand}"},
{"match", override_t::MATCH_EXACT}
}
},
{"remove_matched_tokens", true},
{"filter_by", ""}
};
override_t override_bad1;
op = override_t::parse(override_json_bad1, "dynamic-filters-bad1", override_bad1);
ASSERT_FALSE(op.ok());
ASSERT_EQ("The `filter_by` must be a non-empty string.", op.error());
nlohmann::json override_json_bad2 = {
{"id", "dynamic-filters-bad2"},
{
"rule", {
{"query", "{brand}"},
{"match", override_t::MATCH_EXACT}
}
},
{"remove_matched_tokens", true},
{"filter_by", {"foo", "bar"}}
};
override_t override_bad2;
op = override_t::parse(override_json_bad2, "dynamic-filters-bad2", override_bad2);
ASSERT_FALSE(op.ok());
ASSERT_EQ("The `filter_by` must be a string.", op.error());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionOverrideTest, DynamicFilteringPrefixMatchShouldNotWork) {
Collection *coll1;
std::vector<field> fields = {field("name", field_types::STRING, false),
field("category", field_types::STRING, true),
field("brand", field_types::STRING, true),
field("points", field_types::INT32, false)};
coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "Amazing Shoes";
doc1["category"] = "shoe";
doc1["brand"] = "Nike";
doc1["points"] = 3;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["name"] = "Track Gym";
doc2["category"] = "shoes";
doc2["brand"] = "Adidas";
doc2["points"] = 5;
nlohmann::json doc3;
doc3["id"] = "2";
doc3["name"] = "Running Shoe";
doc3["category"] = "shoes";
doc3["brand"] = "Nike";
doc3["points"] = 5;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
ASSERT_TRUE(coll1->add(doc3.dump()).ok());
std::vector<sort_by> sort_fields = { sort_by("_text_match", "DESC"), sort_by("points", "DESC") };
// with override, results will be different
nlohmann::json override_json = {
{"id", "dynamic-cat-filter"},
{
"rule", {
{"query", "{category}"},
{"match", override_t::MATCH_EXACT}
}
},
{"remove_matched_tokens", true},
{"filter_by", "category: {category}"}
};
override_t override;
auto op = override_t::parse(override_json, "dynamic-cat-filter", override);
ASSERT_TRUE(op.ok());
coll1->add_override(override);
auto results = coll1->search("shoe", {"name", "category", "brand"}, "",
{}, sort_fields, {2, 2, 2}, 10).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionOverrideTest, DynamicFilteringMissingField) {
Collection *coll1;
std::vector<field> fields = {field("name", field_types::STRING, false),
field("category", field_types::STRING, true),
field("points", field_types::INT32, false)};
coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "Amazing Shoes";
doc1["category"] = "shoes";
doc1["points"] = 3;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
std::vector<sort_by> sort_fields = { sort_by("_text_match", "DESC"), sort_by("points", "DESC") };
nlohmann::json override_json = {
{"id", "dynamic-cat-filter"},
{
"rule", {
{"query", "{categories}"}, // this field does NOT exist
{"match", override_t::MATCH_EXACT}
}
},
{"remove_matched_tokens", true},
{"filter_by", "category: {categories}"}
};
override_t override;
auto op = override_t::parse(override_json, "dynamic-cat-filter", override);
ASSERT_TRUE(op.ok());
coll1->add_override(override);
auto results = coll1->search("shoes", {"name", "category"}, "",
{}, sort_fields, {2, 2}, 10).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionOverrideTest, DynamicFilteringBadFilterBy) {
Collection *coll1;
std::vector<field> fields = {field("name", field_types::STRING, false),
field("category", field_types::STRING, true),
field("points", field_types::INT32, false)};
coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "Amazing Shoes";
doc1["category"] = "shoes";
doc1["points"] = 3;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
std::vector<sort_by> sort_fields = { sort_by("_text_match", "DESC"), sort_by("points", "DESC") };
nlohmann::json override_json = {
{"id", "dynamic-cat-filter"},
{
"rule", {
{"query", "{category}"}, // this field does NOT exist
{"match", override_t::MATCH_EXACT}
}
},
{"remove_matched_tokens", true},
{"filter_by", "category: {category} && foo"}
};
override_t override;
auto op = override_t::parse(override_json, "dynamic-cat-filter", override);
ASSERT_TRUE(op.ok());
coll1->add_override(override);
auto results = coll1->search("shoes", {"name", "category"}, "",
{}, sort_fields, {2, 2}, 10).get();
ASSERT_EQ(1, results["hits"].size());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionOverrideTest, DynamicFilteringMultiplePlaceholders) {
Collection* coll1;
std::vector<field> fields = {field("name", field_types::STRING, false),
field("category", field_types::STRING, true),
field("brand", field_types::STRING, true),
field("color", field_types::STRING, true),
field("points", field_types::INT32, false)};
coll1 = collectionManager.get_collection("coll1").get();
if (coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "Retro Shoes";
doc1["category"] = "shoes";
doc1["color"] = "yellow";
doc1["brand"] = "Nike Air Jordan";
doc1["points"] = 3;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["name"] = "Baseball";
doc2["category"] = "shoes";
doc2["color"] = "white";
doc2["brand"] = "Adidas";
doc2["points"] = 5;
nlohmann::json doc3;
doc3["id"] = "2";
doc3["name"] = "Running Shoes";
doc3["category"] = "sports";
doc3["color"] = "grey";
doc3["brand"] = "Nike";
doc3["points"] = 5;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
ASSERT_TRUE(coll1->add(doc3.dump()).ok());
std::vector<sort_by> sort_fields = {sort_by("_text_match", "DESC"), sort_by("points", "DESC")};
nlohmann::json override_json = {
{"id", "dynamic-cat-filter"},
{
"rule", {
{"query", "{brand} {color} shoes"},
{"match", override_t::MATCH_CONTAINS}
}
},
{"remove_matched_tokens", true},
{"filter_by", "brand: {brand} && color: {color}"}
};
override_t override;
auto op = override_t::parse(override_json, "dynamic-cat-filter", override);
ASSERT_TRUE(op.ok());
coll1->add_override(override);
// not an exact match of rule (because of "light") so all results will be fetched, not just Air Jordan brand
auto results = coll1->search("Nike Air Jordan light yellow shoes", {"name", "category", "brand"}, "",
{}, sort_fields, {2, 2, 2}, 10, 1, FREQUENCY, {false}, 10).get();
ASSERT_EQ(3, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("2", results["hits"][1]["document"]["id"].get<std::string>());
ASSERT_EQ("1", results["hits"][2]["document"]["id"].get<std::string>());
// query with tokens at the start that preceding the placeholders in the rule
results = coll1->search("New Nike Air Jordan yellow shoes", {"name", "category", "brand"}, "",
{}, sort_fields, {2, 2, 2}, 10, 1, FREQUENCY, {false}, 10).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionOverrideTest, DynamicFilteringTokensBetweenPlaceholders) {
Collection* coll1;
std::vector<field> fields = {field("name", field_types::STRING, false),
field("category", field_types::STRING, true),
field("brand", field_types::STRING, true),
field("color", field_types::STRING, true),
field("points", field_types::INT32, false)};
coll1 = collectionManager.get_collection("coll1").get();
if (coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "Retro Shoes";
doc1["category"] = "shoes";
doc1["color"] = "yellow";
doc1["brand"] = "Nike Air Jordan";
doc1["points"] = 3;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["name"] = "Baseball";
doc2["category"] = "shoes";
doc2["color"] = "white";
doc2["brand"] = "Adidas";
doc2["points"] = 5;
nlohmann::json doc3;
doc3["id"] = "2";
doc3["name"] = "Running Shoes";
doc3["category"] = "sports";
doc3["color"] = "grey";
doc3["brand"] = "Nike";
doc3["points"] = 5;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
ASSERT_TRUE(coll1->add(doc3.dump()).ok());
std::vector<sort_by> sort_fields = {sort_by("_text_match", "DESC"), sort_by("points", "DESC")};
nlohmann::json override_json = {
{"id", "dynamic-cat-filter"},
{
"rule", {
{"query", "{brand} shoes {color}"},
{"match", override_t::MATCH_CONTAINS}
}
},
{"remove_matched_tokens", true},
{"filter_by", "brand: {brand} && color: {color}"}
};
override_t override;
auto op = override_t::parse(override_json, "dynamic-cat-filter", override);
ASSERT_TRUE(op.ok());
coll1->add_override(override);
auto results = coll1->search("Nike Air Jordan shoes yellow", {"name", "category", "brand"}, "",
{}, sort_fields, {2, 2, 2}, 10).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionOverrideTest, DynamicFilteringWithNumericalFilter) {
Collection* coll1;
std::vector<field> fields = {field("name", field_types::STRING, false),
field("category", field_types::STRING, true),
field("brand", field_types::STRING, true),
field("color", field_types::STRING, true),
field("points", field_types::INT32, false)};
coll1 = collectionManager.get_collection("coll1").get();
if (coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "Retro Shoes";
doc1["category"] = "shoes";
doc1["color"] = "yellow";
doc1["brand"] = "Nike";
doc1["points"] = 15;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["name"] = "Baseball Shoes";
doc2["category"] = "shoes";
doc2["color"] = "white";
doc2["brand"] = "Nike";
doc2["points"] = 5;
nlohmann::json doc3;
doc3["id"] = "2";
doc3["name"] = "Running Shoes";
doc3["category"] = "sports";
doc3["color"] = "grey";
doc3["brand"] = "Nike";
doc3["points"] = 5;
nlohmann::json doc4;
doc4["id"] = "3";
doc4["name"] = "Running Shoes";
doc4["category"] = "sports";
doc4["color"] = "grey";
doc4["brand"] = "Adidas";
doc4["points"] = 5;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
ASSERT_TRUE(coll1->add(doc3.dump()).ok());
ASSERT_TRUE(coll1->add(doc4.dump()).ok());
std::vector<sort_by> sort_fields = {sort_by("_text_match", "DESC"), sort_by("points", "DESC")};
nlohmann::json override_json = {
{"id", "dynamic-cat-filter"},
{
"rule", {
{"query", "popular {brand} shoes"},
{"match", override_t::MATCH_CONTAINS}
}
},
{"remove_matched_tokens", false},
{"filter_by", "brand: {brand} && points:> 10"}
};
override_t override;
auto op = override_t::parse(override_json, "dynamic-cat-filter", override);
ASSERT_TRUE(op.ok());
auto results = coll1->search("popular nike shoes", {"name", "category", "brand"}, "",
{}, sort_fields, {2, 2, 2}, 10, 1, FREQUENCY, {false}, 10).get();
ASSERT_EQ(4, results["hits"].size());
coll1->add_override(override);
results = coll1->search("popular nike shoes", {"name", "category", "brand"}, "",
{}, sort_fields, {2, 2, 2}, 10, 1, FREQUENCY, {false}, 10).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
// when overrides are disabled
bool enable_overrides = false;
results = coll1->search("popular nike shoes", {"name", "category", "brand"}, "",
{}, sort_fields, {2, 2, 2}, 10, 1, FREQUENCY, {false, false, false}, 10,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1, {}, {}, {}, 0,
"<mark>", "</mark>", {1, 1, 1}, 10000, true, false, enable_overrides).get();
ASSERT_EQ(4, results["hits"].size());
// should not match the defined override
results = coll1->search("running adidas shoes", {"name", "category", "brand"}, "",
{}, sort_fields, {2, 2, 2}, 10, 1, FREQUENCY, {false}, 10).get();
ASSERT_EQ(4, results["hits"].size());
ASSERT_EQ("3", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("2", results["hits"][1]["document"]["id"].get<std::string>());
ASSERT_EQ("0", results["hits"][2]["document"]["id"].get<std::string>());
ASSERT_EQ("1", results["hits"][3]["document"]["id"].get<std::string>());
results = coll1->search("adidas", {"name", "category", "brand"}, "",
{}, sort_fields, {2, 2, 2}, 10, 1, FREQUENCY, {false}, 10).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("3", results["hits"][0]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionOverrideTest, DynamicFilteringExactMatch) {
Collection* coll1;
std::vector<field> fields = {field("name", field_types::STRING, false),
field("category", field_types::STRING, true),
field("brand", field_types::STRING, true),
field("color", field_types::STRING, true),
field("points", field_types::INT32, false)};
coll1 = collectionManager.get_collection("coll1").get();
if (coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "Retro Shoes";
doc1["category"] = "shoes";
doc1["color"] = "yellow";
doc1["brand"] = "Nike";
doc1["points"] = 15;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["name"] = "Baseball Shoes";
doc2["category"] = "shoes";
doc2["color"] = "white";
doc2["brand"] = "Nike";
doc2["points"] = 5;
nlohmann::json doc3;
doc3["id"] = "2";
doc3["name"] = "Running Shoes";
doc3["category"] = "sports";
doc3["color"] = "grey";
doc3["brand"] = "Nike";
doc3["points"] = 5;
nlohmann::json doc4;
doc4["id"] = "3";
doc4["name"] = "Running Shoes";
doc4["category"] = "sports";
doc4["color"] = "grey";
doc4["brand"] = "Adidas";
doc4["points"] = 5;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
ASSERT_TRUE(coll1->add(doc3.dump()).ok());
ASSERT_TRUE(coll1->add(doc4.dump()).ok());
std::vector<sort_by> sort_fields = {sort_by("_text_match", "DESC"), sort_by("points", "DESC")};
nlohmann::json override_json = {
{"id", "dynamic-cat-filter"},
{
"rule", {
{"query", "popular {brand} shoes"},
{"match", override_t::MATCH_EXACT}
}
},
{"remove_matched_tokens", false},
{"filter_by", "brand: {brand} && points:> 10"}
};
override_t override;
auto op = override_t::parse(override_json, "dynamic-cat-filter", override);
ASSERT_TRUE(op.ok());
coll1->add_override(override);
auto results = coll1->search("really popular nike shoes", {"name", "category", "brand"}, "",
{}, sort_fields, {2, 2, 2}, 10, 1, FREQUENCY, {true}, 10).get();
ASSERT_EQ(4, results["hits"].size());
results = coll1->search("popular nike running shoes", {"name", "category", "brand"}, "",
{}, sort_fields, {2, 2, 2}, 10, 1, FREQUENCY, {true}, 10).get();
ASSERT_EQ(4, results["hits"].size());
results = coll1->search("popular nike shoes running", {"name", "category", "brand"}, "",
{}, sort_fields, {2, 2, 2}, 10, 1, FREQUENCY, {true}, 10).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("3", results["hits"][1]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionOverrideTest, DynamicFilteringWithSynonyms) {
Collection *coll1;
std::vector<field> fields = {field("name", field_types::STRING, false),
field("category", field_types::STRING, true),
field("brand", field_types::STRING, true),
field("points", field_types::INT32, false)};
coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "Amazing Shoes";
doc1["category"] = "shoes";
doc1["brand"] = "Nike";
doc1["points"] = 3;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["name"] = "Exciting Track Gym";
doc2["category"] = "shoes";
doc2["brand"] = "Adidas";
doc2["points"] = 5;
nlohmann::json doc3;
doc3["id"] = "2";
doc3["name"] = "Amazing Sneakers";
doc3["category"] = "sneakers";
doc3["brand"] = "Adidas";
doc3["points"] = 4;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
ASSERT_TRUE(coll1->add(doc3.dump()).ok());
coll1->add_synonym(R"({"id": "sneakers-shoes", "root": "sneakers", "synonyms": ["shoes"]})"_json);
coll1->add_synonym(R"({"id": "boots-shoes", "root": "boots", "synonyms": ["shoes"]})"_json);
coll1->add_synonym(R"({"id": "exciting-amazing", "root": "exciting", "synonyms": ["amazing"]})"_json);
std::vector<sort_by> sort_fields = { sort_by("_text_match", "DESC"), sort_by("points", "DESC") };
// spaces around field name should still work e.g. "{ field }"
nlohmann::json override_json1 = {
{"id", "dynamic-filters"},
{
"rule", {
{"query", "{ category }"},
{"match", override_t::MATCH_EXACT}
}
},
{"filter_by", "category: {category}"}
};
override_t override1;
auto op = override_t::parse(override_json1, "dynamic-filters", override1);
ASSERT_TRUE(op.ok());
coll1->add_override(override1);
std::map<std::string, override_t*> overrides = coll1->get_overrides().get();
ASSERT_EQ(1, overrides.size());
auto override_json = overrides.at("dynamic-filters")->to_json();
ASSERT_EQ("category: {category}", override_json["filter_by"].get<std::string>());
ASSERT_EQ(true, override_json["remove_matched_tokens"].get<bool>()); // must be true by default
nlohmann::json override_json2 = {
{"id", "static-filters"},
{
"rule", {
{"query", "exciting"},
{"match", override_t::MATCH_CONTAINS}
}
},
{"remove_matched_tokens", true},
{"filter_by", "points: [5, 4]"}
};
override_t override2;
op = override_t::parse(override_json2, "static-filters", override2);
ASSERT_TRUE(op.ok());
coll1->add_override(override2);
auto results = coll1->search("sneakers", {"name", "category", "brand"}, "",
{}, sort_fields, {2, 2, 2}, 10).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
// keyword does not exist but has a synonym with results
results = coll1->search("boots", {"name", "category", "brand"}, "",
{}, sort_fields, {2, 2, 2}, 10).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
// keyword has no override, but synonym's override is used
results = coll1->search("exciting", {"name", "category", "brand"}, "",
{}, sort_fields, {2, 2, 2}, 10).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("2", results["hits"][1]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionOverrideTest, StaticFiltering) {
Collection *coll1;
std::vector<field> fields = {field("name", field_types::STRING, false),
field("price", field_types::FLOAT, true),
field("points", field_types::INT32, false)};
coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "Amazing Shoes";
doc1["price"] = 399.99;
doc1["points"] = 3;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["name"] = "Track Shoes";
doc2["price"] = 49.99;
doc2["points"] = 5;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
std::vector<sort_by> sort_fields = { sort_by("_text_match", "DESC"), sort_by("points", "DESC") };
nlohmann::json override_json_contains = {
{"id", "static-filters"},
{
"rule", {
{"query", "expensive"},
{"match", override_t::MATCH_CONTAINS}
}
},
{"remove_matched_tokens", true},
{"filter_by", "price:> 100"}
};
override_t override_contains;
auto op = override_t::parse(override_json_contains, "static-filters", override_contains);
ASSERT_TRUE(op.ok());
coll1->add_override(override_contains);
nlohmann::json override_json_exact = {
{"id", "static-exact-filters"},
{
"rule", {
{"query", "cheap"},
{"match", override_t::MATCH_EXACT}
}
},
{"remove_matched_tokens", true},
{"filter_by", "price:< 100"}
};
override_t override_exact;
op = override_t::parse(override_json_exact, "static-exact-filters", override_exact);
ASSERT_TRUE(op.ok());
coll1->add_override(override_exact);
auto results = coll1->search("expensive shoes", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
results = coll1->search("expensive", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
// partial word should not match
results = coll1->search("inexpensive shoes", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 10).get();
ASSERT_EQ(2, results["found"].get<uint32_t>());
ASSERT_EQ(2, results["hits"].size());
// with exact match
results = coll1->search("cheap", {"name"}, "",
{}, sort_fields, {2}, 10).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
// should not work in match contains context
results = coll1->search("cheap boots", {"name"}, "",
{}, sort_fields, {2}, 10).get();
ASSERT_EQ(0, results["hits"].size());
// with synonym for expensive: should NOT match as synonyms are resolved after override substitution
coll1->add_synonym(R"({"id": "costly-expensive", "root": "costly", "synonyms": ["expensive"]})"_json);
results = coll1->search("costly", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
ASSERT_EQ(0, results["hits"].size());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionOverrideTest, StaticFilteringMultipleRuleMatch) {
Collection *coll1;
std::vector<field> fields = {field("name", field_types::STRING, false),
field("tags", field_types::STRING_ARRAY, true),
field("points", field_types::INT32, false)};
coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "Amazing Shoes";
doc1["tags"] = {"twitter"};
doc1["points"] = 3;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["name"] = "Track Shoes";
doc2["tags"] = {"starred"};
doc2["points"] = 5;
nlohmann::json doc3;
doc3["id"] = "2";
doc3["name"] = "Track Shoes";
doc3["tags"] = {"twitter", "starred"};
doc3["points"] = 10;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
ASSERT_TRUE(coll1->add(doc3.dump()).ok());
std::vector<sort_by> sort_fields = { sort_by("_text_match", "DESC"), sort_by("points", "DESC") };
nlohmann::json override_filter1_json = {
{"id", "static-filter-1"},
{
"rule", {
{"query", "twitter"},
{"match", override_t::MATCH_CONTAINS}
}
},
{"remove_matched_tokens", true},
{"stop_processing", false},
{"filter_by", "tags: twitter"}
};
override_t override_filter1;
auto op = override_t::parse(override_filter1_json, "static-filter-1", override_filter1);
ASSERT_TRUE(op.ok());
coll1->add_override(override_filter1);
nlohmann::json override_filter2_json = {
{"id", "static-filter-2"},
{
"rule", {
{"query", "starred"},
{"match", override_t::MATCH_CONTAINS}
}
},
{"remove_matched_tokens", true},
{"stop_processing", false},
{"filter_by", "tags: starred"}
};
override_t override_filter2;
op = override_t::parse(override_filter2_json, "static-filter-2", override_filter2);
ASSERT_TRUE(op.ok());
coll1->add_override(override_filter2);
auto results = coll1->search("starred twitter", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
// when stop_processing is enabled (default is true)
override_filter1_json.erase("stop_processing");
override_filter2_json.erase("stop_processing");
override_t override_filter1_reset;
op = override_t::parse(override_filter1_json, "static-filter-1", override_filter1_reset);
ASSERT_TRUE(op.ok());
override_t override_filter2_reset;
op = override_t::parse(override_filter2_json, "static-filter-2", override_filter2_reset);
ASSERT_TRUE(op.ok());
coll1->add_override(override_filter1_reset);
coll1->add_override(override_filter2_reset);
results = coll1->search("starred twitter", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
ASSERT_EQ(0, results["hits"].size());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionOverrideTest, DynamicFilteringMultipleRuleMatch) {
Collection *coll1;
std::vector<field> fields = {field("name", field_types::STRING, false),
field("brand", field_types::STRING, false),
field("tags", field_types::STRING_ARRAY, true),
field("points", field_types::INT32, false)};
coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "Amazing Shoes";
doc1["brand"] = "Nike";
doc1["tags"] = {"twitter"};
doc1["points"] = 3;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["name"] = "Track Shoes";
doc2["brand"] = "Adidas";
doc2["tags"] = {"starred"};
doc2["points"] = 5;
nlohmann::json doc3;
doc3["id"] = "2";
doc3["name"] = "Track Shoes";
doc3["brand"] = "Nike";
doc3["tags"] = {"twitter", "starred"};
doc3["points"] = 10;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
ASSERT_TRUE(coll1->add(doc3.dump()).ok());
std::vector<sort_by> sort_fields = { sort_by("_text_match", "DESC"), sort_by("points", "DESC") };
nlohmann::json override_filter1_json = {
{"id", "dynamic-filter-1"},
{
"rule", {
{"query", "{brand}"},
{"match", override_t::MATCH_CONTAINS}
}
},
{"remove_matched_tokens", true},
{"stop_processing", false},
{"filter_by", "tags: twitter"},
{"metadata", {{"foo", "bar"}}},
};
override_t override_filter1;
auto op = override_t::parse(override_filter1_json, "dynamic-filter-1", override_filter1);
ASSERT_TRUE(op.ok());
coll1->add_override(override_filter1);
ASSERT_EQ("bar", override_filter1.to_json()["metadata"]["foo"].get<std::string>());
nlohmann::json override_filter2_json = {
{"id", "dynamic-filter-2"},
{
"rule", {
{"query", "{tags}"},
{"match", override_t::MATCH_CONTAINS}
}
},
{"remove_matched_tokens", true},
{"stop_processing", false},
{"filter_by", "tags: starred"}
};
override_t override_filter2;
op = override_t::parse(override_filter2_json, "dynamic-filter-2", override_filter2);
ASSERT_TRUE(op.ok());
coll1->add_override(override_filter2);
auto results = coll1->search("starred nike", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("bar", results["metadata"]["foo"].get<std::string>());
// when stop_processing is enabled (default is true)
override_filter1_json.erase("stop_processing");
override_filter2_json.erase("stop_processing");
override_t override_filter1_reset;
op = override_t::parse(override_filter1_json, "dynamic-filter-1", override_filter1_reset);
ASSERT_TRUE(op.ok());
override_t override_filter2_reset;
op = override_t::parse(override_filter2_json, "dynamic-filter-2", override_filter2_reset);
ASSERT_TRUE(op.ok());
coll1->add_override(override_filter1_reset);
coll1->add_override(override_filter2_reset);
results = coll1->search("starred nike", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
ASSERT_EQ(0, results["hits"].size());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionOverrideTest, SynonymsAppliedToOverridenQuery) {
Collection *coll1;
std::vector<field> fields = {field("name", field_types::STRING, false),
field("price", field_types::FLOAT, true),
field("points", field_types::INT32, false)};
coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "Amazing Shoes";
doc1["price"] = 399.99;
doc1["points"] = 3;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["name"] = "White Sneakers";
doc2["price"] = 149.99;
doc2["points"] = 5;
nlohmann::json doc3;
doc3["id"] = "2";
doc3["name"] = "Red Sneakers";
doc3["price"] = 49.99;
doc3["points"] = 5;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
ASSERT_TRUE(coll1->add(doc3.dump()).ok());
std::vector<sort_by> sort_fields = { sort_by("_text_match", "DESC"), sort_by("points", "DESC") };
nlohmann::json override_json_contains = {
{"id", "static-filters"},
{
"rule", {
{"query", "expensive"},
{"match", override_t::MATCH_CONTAINS}
}
},
{"remove_matched_tokens", true},
{"filter_by", "price:> 100"}
};
override_t override_contains;
auto op = override_t::parse(override_json_contains, "static-filters", override_contains);
ASSERT_TRUE(op.ok());
coll1->add_override(override_contains);
coll1->add_synonym(R"({"id": "", "root": "shoes", "synonyms": ["sneakers"]})"_json);
auto results = coll1->search("expensive shoes", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionOverrideTest, StaticFilterWithAndWithoutQueryStringMutation) {
Collection *coll1;
std::vector<field> fields = {field("name", field_types::STRING, false),
field("price", field_types::FLOAT, true),
field("points", field_types::INT32, false)};
coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "Apple iPad";
doc1["price"] = 399.99;
doc1["points"] = 3;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["name"] = "Samsung Charger";
doc2["price"] = 49.99;
doc2["points"] = 5;
nlohmann::json doc3;
doc3["id"] = "2";
doc3["name"] = "Samsung Phone";
doc3["price"] = 249.99;
doc3["points"] = 5;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
ASSERT_TRUE(coll1->add(doc3.dump()).ok());
std::vector<sort_by> sort_fields = { sort_by("_text_match", "DESC"), sort_by("points", "DESC") };
nlohmann::json override_json_contains = {
{"id", "static-filters"},
{
"rule", {
{"query", "apple"},
{"match", override_t::MATCH_CONTAINS}
}
},
{"remove_matched_tokens", false},
{"filter_by", "price:> 200"}
};
override_t override_contains;
auto op = override_t::parse(override_json_contains, "static-filters", override_contains);
ASSERT_TRUE(op.ok());
coll1->add_override(override_contains);
// first without query string mutation
auto results = coll1->search("apple", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
// now, with query string mutation
override_json_contains = {
{"id", "static-filters"},
{
"rule", {
{"query", "apple"},
{"match", override_t::MATCH_CONTAINS}
}
},
{"remove_matched_tokens", true},
{"filter_by", "price:> 200"}
};
op = override_t::parse(override_json_contains, "static-filters", override_contains);
ASSERT_TRUE(op.ok());
coll1->add_override(override_contains);
results = coll1->search("apple", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionOverrideTest, DynamicFilteringWithJustRemoveTokens) {
Collection* coll1;
std::vector<field> fields = {field("name", field_types::STRING, false),
field("category", field_types::STRING, true),
field("brand", field_types::STRING, true),
field("points", field_types::INT32, false)};
coll1 = collectionManager.get_collection("coll1").get();
if (coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "Amazing Shoes";
doc1["category"] = "shoes";
doc1["brand"] = "Nike";
doc1["points"] = 3;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["name"] = "Track Gym";
doc2["category"] = "shoes";
doc2["brand"] = "Adidas";
doc2["points"] = 5;
nlohmann::json doc3;
doc3["id"] = "2";
doc3["name"] = "Running Shoes";
doc3["category"] = "sports";
doc3["brand"] = "Nike";
doc3["points"] = 5;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
ASSERT_TRUE(coll1->add(doc3.dump()).ok());
std::vector<sort_by> sort_fields = {sort_by("_text_match", "DESC"), sort_by("points", "DESC")};
auto results = coll1->search("all", {"name", "category", "brand"}, "",
{}, sort_fields, {0, 0, 0}, 10).get();
ASSERT_EQ(0, results["hits"].size());
// with override, we return all records
nlohmann::json override_json = {
{"id", "match-all"},
{
"rule", {
{"query", "all"},
{"match", override_t::MATCH_EXACT}
}
},
{"remove_matched_tokens", true}
};
override_t override;
auto op = override_t::parse(override_json, "match-all", override);
ASSERT_TRUE(op.ok());
coll1->add_override(override);
results = coll1->search("all", {"name", "category", "brand"}, "",
{}, sort_fields, {0, 0, 0}, 10).get();
ASSERT_EQ(3, results["hits"].size());
results = coll1->search("really amazing shoes", {"name", "category", "brand"}, "",
{}, sort_fields, {0, 0, 0}, 0).get();
ASSERT_EQ(0, results["hits"].size());
// with contains
override_json = {
{"id", "remove-some-tokens"},
{
"rule", {
{"query", "really"},
{"match", override_t::MATCH_CONTAINS}
}
},
{"remove_matched_tokens", true}
};
override_t override2;
op = override_t::parse(override_json, "remove-some-tokens", override2);
ASSERT_TRUE(op.ok());
coll1->add_override(override2);
results = coll1->search("really amazing shoes", {"name", "category", "brand"}, "",
{}, sort_fields, {0, 0, 0}, 1).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionOverrideTest, StaticSorting) {
Collection *coll1;
std::vector<field> fields = {field("name", field_types::STRING, false),
field("price", field_types::FLOAT, true),
field("points", field_types::INT32, false)};
coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "Amazing Shoes";
doc1["price"] = 399.99;
doc1["points"] = 3;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["name"] = "Track Shoes";
doc2["price"] = 49.99;
doc2["points"] = 5;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
std::vector<sort_by> sort_fields = { sort_by("_text_match", "DESC"), sort_by("points", "DESC") };
nlohmann::json override_json_contains = {
{"id", "static-sort"},
{
"rule", {
{"query", "shoes"},
{"match", override_t::MATCH_CONTAINS}
}
},
{"remove_matched_tokens", true},
{"sort_by", "price:desc"}
};
override_t override_contains;
auto op = override_t::parse(override_json_contains, "static-sort", override_contains);
ASSERT_TRUE(op.ok());
// without override kicking in
auto results = coll1->search("shoes", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
// now add override
coll1->add_override(override_contains);
results = coll1->search("shoes", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
// with override we will sort on price
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionOverrideTest, DynamicFilteringWithPartialTokenMatch) {
// when query tokens do not match placeholder field value exactly, don't do filtering
Collection* coll1;
std::vector<field> fields = {field("name", field_types::STRING, false),
field("category", field_types::STRING, true),};
coll1 = collectionManager.get_collection("coll1").get();
if (coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields).get();
}
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "Amazing Shoes";
doc1["category"] = "Running Shoes";
nlohmann::json doc2;
doc2["id"] = "1";
doc2["name"] = "Magic Lamp";
doc2["category"] = "Shoo";
nlohmann::json doc3;
doc3["id"] = "2";
doc3["name"] = "Shox and Us";
doc3["category"] = "Socks";
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
ASSERT_TRUE(coll1->add(doc3.dump()).ok());
std::vector<sort_by> sort_fields = {sort_by("_text_match", "DESC")};
auto results = coll1->search("shoes", {"name"}, "",
{}, sort_fields, {0}, 10).get();
ASSERT_EQ(1, results["hits"].size());
// with override, we return all records
nlohmann::json override_json = {
{"id", "dynamic-filter"},
{
"rule", {
{"query", "{ category }"},
{"match", override_t::MATCH_EXACT}
}
},
{"filter_by", "category:= {category}"},
{"remove_matched_tokens", true}
};
override_t override;
auto op = override_t::parse(override_json, "dynamic-filter", override);
ASSERT_TRUE(op.ok());
coll1->add_override(override);
results = coll1->search("shoes", {"name"}, "",
{}, sort_fields, {0}, 10).get();
ASSERT_EQ(1, results["hits"].size());
results = coll1->search("shox", {"name"}, "",
{}, sort_fields, {0}, 10).get();
ASSERT_EQ(1, results["hits"].size());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionOverrideTest, OverrideWithSymbolsToIndex) {
Collection* coll1;
std::vector<field> fields = {field("name", field_types::STRING, false),
field("category", field_types::STRING, true),};
coll1 = collectionManager.get_collection("coll1").get();
if (coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "", static_cast<uint64_t>(std::time(nullptr)),
"", {"-"}, {}).get();
}
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "Non-Stick";
doc1["category"] = "Cookware";
nlohmann::json doc2;
doc2["id"] = "1";
doc2["name"] = "NonStick";
doc2["category"] = "Kitchen";
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
std::vector<sort_by> sort_fields = {sort_by("_text_match", "DESC")};
auto results = coll1->search("non-stick", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
"", 10).get();
ASSERT_EQ(2, results["hits"].size());
// with override, we return all records
nlohmann::json override_json = {
{"id", "ov-1"},
{
"rule", {
{"query", "non-stick"},
{"match", override_t::MATCH_EXACT}
}
},
{"filter_by", "category:= Cookware"}
};
override_t override;
auto op = override_t::parse(override_json, "ov-1", override, "", {'-'}, {});
ASSERT_TRUE(op.ok());
coll1->add_override(override);
results = coll1->search("non-stick", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
"", 10).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
results = coll1->search("nonstick", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
"", 10).get();
ASSERT_EQ(2, results["hits"].size());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionOverrideTest, OverrideWithTags) {
/*
If override1 is tagged tagA, tagB, override2 is tagged tagA, override3 is tagged with nothing:
Then if a search is tagged with tagA, we only consider overrides that contain tagA (override1 and override2)
with the usual logic - in alphabetic order of override name and then process both if stop rule processing is false.
If a search is tagged with tagA and tagB, we evaluate any rules that contain tagA and tagB first,
then tag A or tag B, but not overrides that contain no tags. Within each group, we evaluate in alphabetic order
and process multiple if stop rule processing is false
If a search has no tags, then we only consider rules that have no tags.
*/
Collection* coll1;
std::vector<field> fields = {field("name", field_types::STRING, false),
field("category", field_types::STRING, true),};
coll1 = collectionManager.get_collection("coll1").get();
if (coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "").get();
}
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "queryA";
doc1["category"] = "kids";
nlohmann::json doc2;
doc2["id"] = "1";
doc2["name"] = "queryA";
doc2["category"] = "kitchen";
nlohmann::json doc3;
doc3["id"] = "2";
doc3["name"] = "Clay Toy";
doc3["category"] = "home";
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
ASSERT_TRUE(coll1->add(doc3.dump()).ok());
std::vector<sort_by> sort_fields = {sort_by("_text_match", "DESC")};
auto results = coll1->search("Clay", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
"", 10).get();
ASSERT_EQ(1, results["hits"].size());
// create overrides containing 2 tags, single tag and no tags:
nlohmann::json override_json1 = R"({
"id": "ov-1",
"rule": {
"query": "queryA",
"match": "exact",
"tags": ["alpha", "beta"]
},
"filter_by": "category: kids"
})"_json;
override_t override1;
auto op = override_t::parse(override_json1, "ov-1", override1);
ASSERT_TRUE(op.ok());
coll1->add_override(override1);
// single tag
nlohmann::json override_json2 = R"({
"id": "ov-2",
"rule": {
"query": "queryA",
"match": "exact",
"tags": ["alpha"]
},
"filter_by": "category: kitchen"
})"_json;
override_t override2;
override_t::parse(override_json2, "ov-2", override2);
ASSERT_TRUE(op.ok());
coll1->add_override(override2);
// no tag
nlohmann::json override_json3 = R"({
"id": "ov-3",
"rule": {
"query": "queryA",
"match": "exact"
},
"filter_by": "category: home"
})"_json;
override_t override3;
op = override_t::parse(override_json3, "ov-3", override3);
ASSERT_TRUE(op.ok());
coll1->add_override(override3);
// when tag doesn't match any override, no results will be found
results = coll1->search("queryA", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 10000,
4, 7, fallback, 4, {off}, 100, 100, 2, 2, false, "", true, 0, max_score, 100, 0,
0, "exhaustive", 30000, 2, "", {}, {}, "right_to_left",
true, true, false, "", "", "foo").get();
ASSERT_EQ(2, results["hits"].size());
// when multiple overrides match a given tag, return first matching record
results = coll1->search("queryA", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 10000,
4, 7, fallback, 4, {off}, 100, 100, 2, 2, false, "", true, 0, max_score, 100, 0,
0, "exhaustive", 30000, 2, "", {}, {}, "right_to_left",
true, true, false, "", "", "alpha").get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
// single tag matching rule with multiple tags
results = coll1->search("queryA", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 10000,
4, 7, fallback, 4, {off}, 100, 100, 2, 2, false, "", true, 0, max_score, 100, 0,
0, "exhaustive", 30000, 2, "", {}, {}, "right_to_left",
true, true, false, "", "", "beta").get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
// when multiple tags are passed, only consider rule with both tags
results = coll1->search("queryA", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 10000,
4, 7, fallback, 4, {off}, 100, 100, 2, 2, false, "", true, 0, max_score, 100, 0,
0, "exhaustive", 30000, 2, "", {}, {}, "right_to_left",
true, true, false, "", "", "alpha,beta").get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
// query with no tags should only trigger override with no tags
results = coll1->search("queryA", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 10000,
4, 7, fallback, 4, {off}, 100, 100, 2, 2, false, "", true, 0, max_score, 100, 0,
0, "exhaustive", 30000, 2, "", {}, {}, "right_to_left",
true, true, false, "", "", "").get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionOverrideTest, OverrideWithTagsPartialMatch) {
Collection* coll1;
std::vector<field> fields = {field("name", field_types::STRING, false),
field("category", field_types::STRING, true),};
coll1 = collectionManager.get_collection("coll1").get();
if (coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "").get();
}
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "queryA";
doc1["category"] = "kids";
nlohmann::json doc2;
doc2["id"] = "1";
doc2["name"] = "queryA";
doc2["category"] = "kitchen";
nlohmann::json doc3;
doc3["id"] = "2";
doc3["name"] = "Clay Toy";
doc3["category"] = "home";
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
ASSERT_TRUE(coll1->add(doc3.dump()).ok());
std::vector<sort_by> sort_fields = {sort_by("_text_match", "DESC")};
nlohmann::json override_json1 = R"({
"id": "ov-1",
"rule": {
"query": "queryA",
"match": "exact",
"tags": ["alpha", "beta"]
},
"filter_by": "category: kids"
})"_json;
override_t override1;
auto op = override_t::parse(override_json1, "ov-1", override1);
ASSERT_TRUE(op.ok());
coll1->add_override(override1);
//
nlohmann::json override_json2 = R"({
"id": "ov-2",
"rule": {
"query": "queryB",
"match": "exact",
"tags": ["alpha"]
},
"filter_by": "category: kitchen"
})"_json;
override_t override2;
override_t::parse(override_json2, "ov-2", override2);
ASSERT_TRUE(op.ok());
coll1->add_override(override2);
// when only one of the two tags are found, apply that rule
auto results = coll1->search("queryB", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 10000,
4, 7, fallback, 4, {off}, 100, 100, 2, 2, false, "", true, 0, max_score, 100, 0,
0, "exhaustive", 30000, 2, "", {}, {}, "right_to_left",
true, true, false, "", "", "alpha,zeta").get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionOverrideTest, OverrideWithTagsWithoutStopProcessing) {
Collection* coll1;
std::vector<field> fields = {field("name", field_types::STRING, false),
field("category", field_types::STRING_ARRAY, true),};
coll1 = collectionManager.get_collection("coll1").get();
if (coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "").get();
}
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "queryA";
doc1["category"] = {"kids"};
nlohmann::json doc2;
doc2["id"] = "1";
doc2["name"] = "queryA";
doc2["category"] = {"kids", "kitchen"};
nlohmann::json doc3;
doc3["id"] = "2";
doc3["name"] = "Clay Toy";
doc3["category"] = {"home"};
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
ASSERT_TRUE(coll1->add(doc3.dump()).ok());
std::vector<sort_by> sort_fields = {sort_by("_text_match", "DESC")};
nlohmann::json override_json1 = R"({
"id": "ov-1",
"rule": {
"query": "queryA",
"match": "exact",
"tags": ["alpha", "beta"]
},
"stop_processing": false,
"remove_matched_tokens": false,
"filter_by": "category: kids"
})"_json;
override_t override1;
auto op = override_t::parse(override_json1, "ov-1", override1);
ASSERT_TRUE(op.ok());
coll1->add_override(override1);
//
nlohmann::json override_json2 = R"({
"id": "ov-2",
"rule": {
"query": "queryA",
"match": "exact",
"tags": ["alpha"]
},
"stop_processing": false,
"remove_matched_tokens": false,
"filter_by": "category: kitchen",
"metadata": {"foo": "bar"}
})"_json;
override_t override2;
override_t::parse(override_json2, "ov-2", override2);
ASSERT_TRUE(op.ok());
coll1->add_override(override2);
//
nlohmann::json override_json3 = R"({
"id": "ov-3",
"rule": {
"query": "queryA",
"match": "exact"
},
"stop_processing": false,
"remove_matched_tokens": false,
"filter_by": "category: home"
})"_json;
override_t override3;
op = override_t::parse(override_json3, "ov-3", override3);
ASSERT_TRUE(op.ok());
coll1->add_override(override3);
auto results = coll1->search("queryA", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 10000,
4, 7, fallback, 4, {off}, 100, 100, 2, 2, false, "", true, 0, max_score, 100, 0,
0, "exhaustive", 30000, 2, "", {}, {}, "right_to_left",
true, true, false, "", "", "alpha").get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("bar", results["metadata"]["foo"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionOverrideTest, WildcardTagRuleThatMatchesAllQueries) {
Collection* coll1;
std::vector<field> fields = {field("name", field_types::STRING, false),
field("category", field_types::STRING, true),};
coll1 = collectionManager.get_collection("coll1").get();
if (coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "").get();
}
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "queryA";
doc1["category"] = "kids";
nlohmann::json doc2;
doc2["id"] = "1";
doc2["name"] = "queryA";
doc2["category"] = "kitchen";
nlohmann::json doc3;
doc3["id"] = "2";
doc3["name"] = "Clay Toy";
doc3["category"] = "home";
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
ASSERT_TRUE(coll1->add(doc3.dump()).ok());
std::vector<sort_by> sort_fields = {sort_by("_text_match", "DESC")};
nlohmann::json override_json1 = R"({
"id": "ov-1",
"rule": {
},
"filter_by": "category: kids"
})"_json;
override_t override1;
auto op = override_t::parse(override_json1, "ov-1", override1);
ASSERT_FALSE(op.ok());
ASSERT_EQ("The `rule` definition must contain either a `tags` or a `query` and `match`.", op.error());
override_json1 = R"({
"id": "ov-1",
"rule": {
"tags": ["*"]
},
"filter_by": "category: kids"
})"_json;
op = override_t::parse(override_json1, "ov-1", override1);
ASSERT_TRUE(op.ok());
coll1->add_override(override1);
// should match all search queries, even without passing any tags
std::string override_tags = "";
auto results = coll1->search("queryB", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 10000,
4, 7, fallback, 4, {off}, 100, 100, 2, 2, false, "", true, 0, max_score, 100, 0,
0, "exhaustive", 30000, 2, "", {}, {}, "right_to_left",
true, true, false, "", "", override_tags).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
results = coll1->search("queryA", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 10000,
4, 7, fallback, 4, {off}, 100, 100, 2, 2, false, "", true, 0, max_score, 100, 0,
0, "exhaustive", 30000, 2, "", {}, {}, "right_to_left",
true, true, false, "", "", override_tags).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
// includes instead of filter_by
coll1->remove_override("ov-1");
auto override_json2 = R"({
"id": "ov-1",
"rule": {
"tags": ["*"]
},
"includes": [
{"id": "1", "position": 1}
]
})"_json;
override_t override2;
op = override_t::parse(override_json2, "ov-2", override2);
ASSERT_TRUE(op.ok());
coll1->add_override(override2);
results = coll1->search("foobar", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 10000,
4, 7, fallback, 4, {off}, 100, 100, 2, 2, false, "", true, 0, max_score, 100, 0,
0, "exhaustive", 30000, 2, "", {}, {}, "right_to_left",
true, true, false, "", "", override_tags).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionOverrideTest, TagsOnlyRule) {
Collection* coll1;
std::vector<field> fields = {field("name", field_types::STRING, false),
field("category", field_types::STRING_ARRAY, true),};
coll1 = collectionManager.get_collection("coll1").get();
if (coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "").get();
}
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "queryA";
doc1["category"] = {"kids"};
nlohmann::json doc2;
doc2["id"] = "1";
doc2["name"] = "queryA";
doc2["category"] = {"kitchen"};
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
std::vector<sort_by> sort_fields = {sort_by("_text_match", "DESC")};
override_t override1;
auto override_json1 = R"({
"id": "ov-1",
"rule": {
"tags": ["listing"]
},
"filter_by": "category: kids"
})"_json;
auto op = override_t::parse(override_json1, "ov-1", override1);
ASSERT_TRUE(op.ok());
coll1->add_override(override1);
auto results = coll1->search("queryA", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 10000,
4, 7, fallback, 4, {off}, 100, 100, 2, 2, false, "", true, 0, max_score, 100, 0,
0, "exhaustive", 30000, 2, "", {}, {}, "right_to_left",
true, true, false, "", "", "listing").get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
// with include rule
override_t override2;
auto override_json2 = R"({
"id": "ov-2",
"rule": {
"tags": ["listing2"]
},
"includes": [
{"id": "1", "position": 1}
]
})"_json;
op = override_t::parse(override_json2, "ov-2", override2);
ASSERT_TRUE(op.ok());
coll1->add_override(override2);
results = coll1->search("foobar", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 10000,
4, 7, fallback, 4, {off}, 100, 100, 2, 2, false, "", true, 0, max_score, 100, 0,
0, "exhaustive", 30000, 2, "", {}, {}, "right_to_left",
true, true, false, "", "", "listing2").get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
// no override tag passed: rule should not match
std::string override_tag = "";
results = coll1->search("foobar", {"name"}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 10000,
4, 7, fallback, 4, {off}, 100, 100, 2, 2, false, "", true, 0, max_score, 100, 0,
0, "exhaustive", 30000, 2, "", {}, {}, "right_to_left",
true, true, false, "", "", override_tag).get();
ASSERT_EQ(0, results["hits"].size());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionOverrideTest, MetadataValidation) {
Collection* coll1;
std::vector<field> fields = {field("name", field_types::STRING, false),
field("category", field_types::STRING_ARRAY, true),};
coll1 = collectionManager.get_collection("coll1").get();
if (coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "").get();
}
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "queryA";
doc1["category"] = {"kids"};
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
std::vector<sort_by> sort_fields = {sort_by("_text_match", "DESC")};
nlohmann::json override_json1 = R"({
"id": "ov-1",
"rule": {
"query": "queryA",
"match": "exact"
},
"filter_by": "category: kids",
"metadata": "foo"
})"_json;
override_t override1;
auto op = override_t::parse(override_json1, "ov-1", override1);
ASSERT_FALSE(op.ok());
ASSERT_EQ("The `metadata` must be a JSON object.", op.error());
// don't allow empty rule without any action
override_json1 = R"({
"id": "ov-1",
"rule": {
"query": "queryA",
"match": "exact"
}
})"_json;
override_t override2;
op = override_t::parse(override_json1, "ov-2", override2);
ASSERT_FALSE(op.ok());
ASSERT_EQ("Must contain one of: `includes`, `excludes`, `metadata`, `filter_by`, `sort_by`, "
"`remove_matched_tokens`, `replace_query`.", op.error());
// should allow only metadata to be present as action
override_json1 = R"({
"id": "ov-1",
"rule": {
"query": "queryA",
"match": "exact"
},
"metadata": {"foo": "bar"}
})"_json;
override_t override3;
op = override_t::parse(override_json1, "ov-3", override3);
ASSERT_TRUE(op.ok());
coll1->add_override(override3);
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionOverrideTest, WildcardSearchOverride) {
Collection* coll1;
std::vector<field> fields = {field("name", field_types::STRING, false),
field("category", field_types::STRING, true),};
coll1 = collectionManager.get_collection("coll1").get();
if (coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "").get();
}
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "queryA";
doc1["category"] = "kids";
nlohmann::json doc2;
doc2["id"] = "1";
doc2["name"] = "queryA";
doc2["category"] = "kitchen";
nlohmann::json doc3;
doc3["id"] = "2";
doc3["name"] = "Clay Toy";
doc3["category"] = "home";
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
ASSERT_TRUE(coll1->add(doc3.dump()).ok());
std::vector<sort_by> sort_fields = {sort_by("_text_match", "DESC")};
nlohmann::json override_json1 = R"({
"id": "ov-1",
"rule": {
"query": "*",
"match": "exact"
},
"filter_by": "category: kids"
})"_json;
override_t override1;
auto op = override_t::parse(override_json1, "ov-1", override1);
ASSERT_TRUE(op.ok());
coll1->add_override(override1);
std::string override_tags = "";
auto results = coll1->search("*", {}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 10000,
4, 7, fallback, 4, {off}, 100, 100, 2, 2, false, "", true, 0, max_score, 100, 0,
0, "exhaustive", 30000, 2, "", {}, {}, "right_to_left",
true, true, false, "", "", override_tags).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
// includes instead of filter_by
coll1->remove_override("ov-1");
override_t override2;
auto override_json2 = R"({
"id": "ov-2",
"rule": {
"query": "*",
"match": "exact"
},
"includes": [
{"id": "1", "position": 1}
]
})"_json;
op = override_t::parse(override_json2, "ov-2", override2);
ASSERT_TRUE(op.ok());
coll1->add_override(override2);
results = coll1->search("*", {}, "",
{}, sort_fields, {2}, 10, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 10000,
4, 7, fallback, 4, {off}, 100, 100, 2, 2, false, "", true, 0, max_score, 100, 0,
0, "exhaustive", 30000, 2, "", {}, {}, "right_to_left",
true, true, false, "", "", override_tags).get();
ASSERT_EQ(3, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionOverrideTest, OverridesPagination) {
Collection *coll2;
std::vector<field> fields = {field("title", field_types::STRING, false),
field("points", field_types::INT32, false)};
coll2 = collectionManager.get_collection("coll2").get();
if(coll2 == nullptr) {
coll2 = collectionManager.create_collection("coll2", 1, fields, "points").get();
}
for(int i = 0; i < 5; ++i) {
nlohmann::json override_json = {
{"id", "override"},
{
"rule", {
{"query", "not-found"},
{"match", override_t::MATCH_EXACT}
}
},
{"metadata", { {"foo", "bar"}}},
};
override_json["id"] = override_json["id"].get<std::string>() + std::to_string(i + 1);
override_t override;
override_t::parse(override_json, "", override);
coll2->add_override(override);
}
uint32_t limit = 0, offset = 0, i = 0;
//limit collections by 2
limit=2;
auto override_op = coll2->get_overrides(limit);
auto override_map = override_op.get();
ASSERT_EQ(2, override_map.size());
i=offset;
for(const auto &kv : override_map) {
ASSERT_EQ("override" + std::to_string(i+1), kv.second->id);
++i;
}
//get 2 collection from offset 3
offset=3;
override_op = coll2->get_overrides(limit, offset);
override_map = override_op.get();
ASSERT_EQ(2, override_map.size());
i=offset;
for(const auto &kv : override_map) {
ASSERT_EQ("override" + std::to_string(i+1), kv.second->id);
++i;
}
//get all collection except first
offset=1; limit=0;
override_op = coll2->get_overrides(limit, offset);
override_map = override_op.get();
ASSERT_EQ(4, override_map.size());
i=offset;
for(const auto &kv : override_map) {
ASSERT_EQ("override" + std::to_string(i+1), kv.second->id);
++i;
}
//get last collection
offset=4, limit=1;
override_op = coll2->get_overrides(limit, offset);
override_map = override_op.get();
ASSERT_EQ(1, override_map.size());
ASSERT_EQ("override5", override_map.begin()->second->id);
//if limit is greater than number of collection then return all from offset
offset=0; limit=8;
override_op = coll2->get_overrides(limit, offset);
override_map = override_op.get();
ASSERT_EQ(5, override_map.size());
i=offset;
for(const auto &kv : override_map) {
ASSERT_EQ("override" + std::to_string(i+1), kv.second->id);
++i;
}
offset=3; limit=4;
override_op = coll2->get_overrides(limit, offset);
override_map = override_op.get();
ASSERT_EQ(2, override_map.size());
i=offset;
for(const auto &kv : override_map) {
ASSERT_EQ("override" + std::to_string(i+1), kv.second->id);
++i;
}
//invalid offset
offset=6; limit=0;
override_op = coll2->get_overrides(limit, offset);
ASSERT_FALSE(override_op.ok());
ASSERT_EQ("Invalid offset param.", override_op.error());
}
TEST_F(CollectionOverrideTest, RetrieveOverideByID) {
Collection *coll2;
std::vector<field> fields = {field("title", field_types::STRING, false),
field("points", field_types::INT32, false)};
coll2 = collectionManager.get_collection("coll2").get();
if (coll2 == nullptr) {
coll2 = collectionManager.create_collection("coll2", 1, fields, "points").get();
}
nlohmann::json override_json = {
{"id", "override"},
{
"rule", {
{"query", "not-found"},
{"match", override_t::MATCH_EXACT}
}
},
{"metadata", { {"foo", "bar"}}},
};
override_json["id"] = override_json["id"].get<std::string>() + "1";
override_t override;
override_t::parse(override_json, "", override);
coll2->add_override(override);
auto op = coll2->get_override("override1");
ASSERT_TRUE(op.ok());
}
TEST_F(CollectionOverrideTest, FilterPinnedHits) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("points", field_types::INT32, false)};
Collection* coll3 = collectionManager.get_collection("coll3").get();
if(coll3 == nullptr) {
coll3 = collectionManager.create_collection("coll3", 1, fields, "points").get();
}
nlohmann::json doc;
doc["title"] = "Snapdragon 7 gen 2023";
doc["points"] = 100;
ASSERT_TRUE(coll3->add(doc.dump()).ok());
doc["title"] = "Snapdragon 732G 2023";
doc["points"] = 91;
ASSERT_TRUE(coll3->add(doc.dump()).ok());
doc["title"] = "Snapdragon 4 gen 2023";
doc["points"] = 65;
ASSERT_TRUE(coll3->add(doc.dump()).ok());
doc["title"] = "Mediatek Dimensity 720G 2022";
doc["points"] = 87;
ASSERT_TRUE(coll3->add(doc.dump()).ok());
doc["title"] = "Mediatek Dimensity 470G 2023";
doc["points"] = 63;
ASSERT_TRUE(coll3->add(doc.dump()).ok());
auto pinned_hits = "3:1, 4:2";
bool filter_curated_hits = false;
auto results = coll3->search("2023", {"title"}, "title: snapdragon", {}, {},
{0}, 50, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10,
"", 30, 5, "",
10, pinned_hits, {}, {}, 3,
"<mark>", "</mark>", {}, UINT_MAX,
true, false, true, "",
false, 6000 * 1000, 4, 7,
fallback, 4, {off}, INT16_MAX,
INT16_MAX, 2, filter_curated_hits ).get();
ASSERT_EQ(5, results["hits"].size());
ASSERT_EQ("3", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("4", results["hits"][1]["document"]["id"].get<std::string>());
ASSERT_EQ("0", results["hits"][2]["document"]["id"].get<std::string>());
ASSERT_EQ("1", results["hits"][3]["document"]["id"].get<std::string>());
ASSERT_EQ("2", results["hits"][4]["document"]["id"].get<std::string>());
// when filter does not match, we should return only curated results
results = coll3->search("2023", {"title"}, "title: foobarbaz", {}, {},
{0}, 50, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10,
"", 30, 5, "",
10, pinned_hits, {}, {}, 3,
"<mark>", "</mark>", {}, UINT_MAX,
true, false, true, "",
false, 6000 * 1000, 4, 7,
fallback, 4, {off}, INT16_MAX,
INT16_MAX, 2, filter_curated_hits ).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("3", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("4", results["hits"][1]["document"]["id"].get<std::string>());
filter_curated_hits = true;
// Filter should apply on curated results
results = coll3->search("2023", {"title"}, "points: >70", {}, {},
{0}, 50, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10,
"", 30, 5, "",
10, pinned_hits, {}, {}, 3,
"<mark>", "</mark>", {}, UINT_MAX,
true, false, true, "",
false, 6000 * 1000, 4, 7,
fallback, 4, {off}, INT16_MAX,
INT16_MAX, 2, filter_curated_hits ).get();
ASSERT_EQ(3, results["hits"].size());
ASSERT_EQ("3", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
ASSERT_EQ("1", results["hits"][2]["document"]["id"].get<std::string>());
results = coll3->search("2023", {"title"}, "title: snapdragon", {}, {},
{0}, 50, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10,
"", 30, 5, "",
10, pinned_hits, {}, {}, 3,
"<mark>", "</mark>", {}, UINT_MAX,
true, false, true, "",
false, 6000 * 1000, 4, 7,
fallback, 4, {off}, INT16_MAX,
INT16_MAX, 2, filter_curated_hits).get();
ASSERT_EQ(3, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
ASSERT_EQ("2", results["hits"][2]["document"]["id"].get<std::string>());
//partial filter out ids, remaining will take higher precedence than their assignment
results = coll3->search("snapdragon", {"title"}, "title: 2023", {}, {},
{0}, 50, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10,
"", 30, 5, "",
10, pinned_hits, {}, {}, 3,
"<mark>", "</mark>", {}, UINT_MAX,
true, false, true, "",
false, 6000 * 1000, 4, 7,
fallback, 4, {off}, INT16_MAX,
INT16_MAX, 2, filter_curated_hits).get();
ASSERT_EQ(4, results["hits"].size());
ASSERT_EQ("4", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
ASSERT_EQ("1", results["hits"][2]["document"]["id"].get<std::string>());
ASSERT_EQ("2", results["hits"][3]["document"]["id"].get<std::string>());
}
TEST_F(CollectionOverrideTest, AvoidTypoMatchingWhenOverlapWithCuratedData) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("points", field_types::INT32, false)};
Collection* coll3 = collectionManager.get_collection("coll3").get();
if (coll3 == nullptr) {
coll3 = collectionManager.create_collection("coll3", 1, fields, "points").get();
}
nlohmann::json doc;
doc["title"] = "Snapdragon 7 gen 2023";
doc["points"] = 100;
ASSERT_TRUE(coll3->add(doc.dump()).ok());
doc["title"] = "Snapdragon 732G 2023";
doc["points"] = 91;
ASSERT_TRUE(coll3->add(doc.dump()).ok());
doc["title"] = "Mediatak 4 gen 2023";
doc["points"] = 65;
ASSERT_TRUE(coll3->add(doc.dump()).ok());
doc["title"] = "Mediatek Dimensity 720G 2022";
doc["points"] = 87;
ASSERT_TRUE(coll3->add(doc.dump()).ok());
doc["title"] = "Mediatek Dimensity 470G 2023";
doc["points"] = 63;
ASSERT_TRUE(coll3->add(doc.dump()).ok());
auto pinned_hits = "3:1, 4:2";
auto results = coll3->search("Mediatek", {"title"}, "", {}, {},
{2}, 50, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10,
"", 30, 5, "",
10, pinned_hits, {}, {}, 3,
"<mark>", "</mark>", {}, UINT_MAX,
true, false, true, "",
false, 6000 * 1000, 4, 7,
fallback, 4, {off}, INT16_MAX,
INT16_MAX, 2, false).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("3", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("4", results["hits"][1]["document"]["id"].get<std::string>());
results = coll3->search("Mediatek", {"title"}, "", {}, {},
{2}, 50, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10,
"", 30, 5, "",
10, pinned_hits, {}, {}, 3,
"<mark>", "</mark>", {}, UINT_MAX,
true, false, true, "",
false, 6000 * 1000, 4, 7,
fallback, 4, {off}, INT16_MAX,
INT16_MAX, 2, false).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("3", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("4", results["hits"][1]["document"]["id"].get<std::string>());
}
TEST_F(CollectionOverrideTest, OverridesWithSemanticSearch) {
auto schema_json = R"({
"name": "products",
"fields":[
{
"name": "product_name",
"type": "string"
},
{
"name": "embedding",
"type": "float[]",
"embed": {
"from": [
"product_name"
],
"model_config": {
"model_name": "ts/clip-vit-b-p32"
}
}
}
]
})"_json;
EmbedderManager::set_model_dir("/tmp/typesense_test/models");
auto coll_op = collectionManager.create_collection(schema_json);
ASSERT_TRUE(coll_op.ok());
auto coll = coll_op.get();
std::vector<std::string> products = {"Cell Phone", "Laptop", "Desktop", "Printer", "Keyboard", "Monitor", "Mouse"};
nlohmann::json doc;
for (auto product: products) {
doc["product_name"] = product;
ASSERT_TRUE(coll->add(doc.dump()).ok());
}
auto results = coll->search("phone", {"embedding"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
{"embedding"}).get();
ASSERT_EQ(results["found"], 7);
nlohmann::json override_json = {
{"id", "exclude-rule"},
{
"rule", {
{"query", "phone"},
{"match", override_t::MATCH_CONTAINS}
}
}
};
override_json["excludes"] = nlohmann::json::array();
override_json["excludes"][0] = nlohmann::json::object();
override_json["excludes"][0]["id"] = "0";
override_t override;
override_t::parse(override_json, "", override);
ASSERT_TRUE(coll->add_override(override).ok());
results = coll->search("phone", {"embedding"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
{"embedding"}).get();
ASSERT_EQ(results["found"], 6);
ASSERT_EQ(results["hits"][0]["document"]["id"], "4");
ASSERT_EQ(results["hits"][1]["document"]["id"], "6");
ASSERT_EQ(results["hits"][2]["document"]["id"], "1");
ASSERT_EQ(results["hits"][3]["document"]["id"], "5");
ASSERT_EQ(results["hits"][4]["document"]["id"], "2");
ASSERT_EQ(results["hits"][5]["document"]["id"], "3");
}