Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Add response metadata via override.

Kishore Nallanc21c8557032a
diff --git a/include/collection.h b/include/collection.h
index bc949e61..9bfbc6f6 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -194,7 +194,8 @@ private:
std::vector<uint32_t>& excluded_ids,
std::vector<const override_t*>& filter_overrides,
bool& filter_curated_hits,
- std::string& curated_sort_by) const;
+ std::string& curated_sort_by,
+ nlohmann::json& override_metadata) const;
void curate_results(string& actual_query, const string& filter_query, bool enable_overrides, bool already_segmented,
const std::set<std::string>& tags,
@@ -203,7 +204,7 @@ private:
std::vector<std::pair<uint32_t, uint32_t>>& included_ids,
std::vector<uint32_t>& excluded_ids, std::vector<const override_t*>& filter_overrides,
bool& filter_curated_hits,
- std::string& curated_sort_by) const;
+ std::string& curated_sort_by, nlohmann::json& override_metadata) const;
static Option<bool> detect_new_fields(nlohmann::json& document,
const DIRTY_VALUES& dirty_values,
@@ -274,7 +275,8 @@ private:
token_ordering token_order,
filter_node_t*& filter_tree_root,
std::vector<std::pair<uint32_t, uint32_t>>& included_ids,
- std::vector<uint32_t>& excluded_ids) const;
+ std::vector<uint32_t>& excluded_ids,
+ nlohmann::json& override_metadata) const;
void populate_text_match_info(nlohmann::json& info, uint64_t match_score, const text_match_type_t match_type) const;
diff --git a/include/index.h b/include/index.h
index b9805daa..db881e87 100644
--- a/include/index.h
+++ b/include/index.h
@@ -1022,7 +1022,8 @@ public:
std::vector<std::string>& query_tokens,
token_ordering token_order,
filter_node_t*& filter_tree_root,
- std::vector<const override_t*>& matched_dynamic_overrides) const;
+ std::vector<const override_t*>& matched_dynamic_overrides,
+ nlohmann::json& override_metadata) const;
Option<bool> compute_sort_scores(const std::vector<sort_by>& sort_fields, const int* sort_order,
std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3> field_values,
diff --git a/include/override.h b/include/override.h
index 0496bc4b..c19a4639 100644
--- a/include/override.h
+++ b/include/override.h
@@ -40,6 +40,8 @@ struct override_t {
std::string sort_by;
std::string replace_query;
+ nlohmann::json metadata;
+
// epoch seconds
int64_t effective_from_ts = -1;
int64_t effective_to_ts = -1;
diff --git a/src/collection.cpp b/src/collection.cpp
index 7fc281b8..244f339f 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -821,7 +821,8 @@ bool Collection::does_override_match(const override_t& override, std::string& qu
std::vector<uint32_t>& excluded_ids,
std::vector<const override_t*>& filter_overrides,
bool& filter_curated_hits,
- std::string& curated_sort_by) const {
+ std::string& curated_sort_by,
+ nlohmann::json& override_metadata) const {
auto now_epoch = int64_t(std::time(0));
if(override.effective_from_ts != -1 && now_epoch < override.effective_from_ts) {
@@ -888,6 +889,9 @@ bool Collection::does_override_match(const override_t& override, std::string& qu
filter_curated_hits = override.filter_curated_hits;
curated_sort_by = override.sort_by;
+ if(override_metadata.empty()) {
+ override_metadata = override.metadata;
+ }
return true;
}
@@ -900,7 +904,8 @@ void Collection::curate_results(string& actual_query, const string& filter_query
std::vector<uint32_t>& excluded_ids,
std::vector<const override_t*>& filter_overrides,
bool& filter_curated_hits,
- std::string& curated_sort_by) const {
+ std::string& curated_sort_by,
+ nlohmann::json& override_metadata) const {
std::set<uint32_t> excluded_set;
@@ -951,7 +956,7 @@ void Collection::curate_results(string& actual_query, const string& filter_query
filter_query, already_segmented, tags,
pinned_hits, hidden_hits, included_ids,
excluded_ids, filter_overrides, filter_curated_hits,
- curated_sort_by);
+ curated_sort_by, override_metadata);
if(match_found) {
all_tags_found = true;
@@ -998,7 +1003,7 @@ void Collection::curate_results(string& actual_query, const string& filter_query
filter_query, already_segmented, tags,
pinned_hits, hidden_hits, included_ids,
excluded_ids, filter_overrides, filter_curated_hits,
- curated_sort_by);
+ curated_sort_by, override_metadata);
if(match_found) {
found_overrides.insert(id);
@@ -1014,7 +1019,8 @@ void Collection::curate_results(string& actual_query, const string& filter_query
const auto& override = override_kv.second;
bool match_found = does_override_match(override, query, excluded_set, actual_query, filter_query,
already_segmented, tags, pinned_hits, hidden_hits, included_ids,
- excluded_ids, filter_overrides, filter_curated_hits, curated_sort_by);
+ excluded_ids, filter_overrides, filter_curated_hits,
+ curated_sort_by, override_metadata);
if(match_found && override.stop_processing) {
break;
}
@@ -2042,6 +2048,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
std::vector<std::string> hidden_hits;
StringUtils::split(hidden_hits_str, hidden_hits, ",");
+ nlohmann::json override_metadata;
std::vector<const override_t*> filter_overrides;
bool filter_curated_hits = false;
std::string curated_sort_by;
@@ -2055,7 +2062,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
curate_results(query, filter_query, enable_overrides, pre_segmented_query, override_tag_set,
pinned_hits, hidden_hits, included_ids, excluded_ids, filter_overrides, filter_curated_hits,
- curated_sort_by);
+ curated_sort_by, override_metadata);
if(filter_curated_hits_option == 0 || filter_curated_hits_option == 1) {
// When query param has explicit value set, override level configuration takes lower precedence.
@@ -2152,7 +2159,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
// included_ids, excluded_ids
process_filter_overrides(filter_overrides, q_include_tokens, token_order, filter_tree_root,
- included_ids, excluded_ids);
+ included_ids, excluded_ids, override_metadata);
for(size_t i = 0; i < q_include_tokens.size(); i++) {
auto& q_include_token = q_include_tokens[i];
@@ -2888,6 +2895,10 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
result["request_params"]["per_page"] = per_page;
result["request_params"]["q"] = raw_query;
+ if(!override_metadata.empty()) {
+ result["metadata"] = override_metadata;
+ }
+
//long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();
//!LOG(INFO) << "Time taken for result calc: " << timeMillis << "us";
//!store->print_memory_usage();
@@ -3161,11 +3172,12 @@ void Collection::process_filter_overrides(std::vector<const override_t*>& filter
token_ordering token_order,
filter_node_t*& filter_tree_root,
std::vector<std::pair<uint32_t, uint32_t>>& included_ids,
- std::vector<uint32_t>& excluded_ids) const {
+ std::vector<uint32_t>& excluded_ids,
+ nlohmann::json& override_metadata) const {
std::vector<const override_t*> matched_dynamic_overrides;
index->process_filter_overrides(filter_overrides, q_include_tokens, token_order,
- filter_tree_root, matched_dynamic_overrides);
+ filter_tree_root, matched_dynamic_overrides, override_metadata);
// we will check the dynamic overrides to see if they also have include/exclude
std::set<uint32_t> excluded_set;
diff --git a/src/index.cpp b/src/index.cpp
index 573c040c..cf0640d1 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -2130,7 +2130,8 @@ void Index::process_filter_overrides(const std::vector<const override_t*>& filte
std::vector<std::string>& query_tokens,
token_ordering token_order,
filter_node_t*& filter_tree_root,
- std::vector<const override_t*>& matched_dynamic_overrides) const {
+ std::vector<const override_t*>& matched_dynamic_overrides,
+ nlohmann::json& override_metadata) const {
std::shared_lock lock(mutex);
for (auto& override : filter_overrides) {
@@ -2140,6 +2141,9 @@ void Index::process_filter_overrides(const std::vector<const override_t*>& filte
bool resolved_override = static_filter_query_eval(override, query_tokens, filter_tree_root);
if (resolved_override) {
+ if(override_metadata.empty()) {
+ override_metadata = override->metadata;
+ }
if (override->remove_matched_tokens) {
std::vector<std::string> rule_tokens;
Tokenizer(override->rule.query, true).tokenize(rule_tokens);
@@ -2166,6 +2170,10 @@ void Index::process_filter_overrides(const std::vector<const override_t*>& filte
token_order, absorbed_tokens, filter_by_clause);
if (resolved_override) {
+ if(override_metadata.empty()) {
+ override_metadata = override->metadata;
+ }
+
filter_node_t* new_filter_tree_root = nullptr;
Option<bool> filter_op = filter::parse_filter_query(filter_by_clause, search_schema,
store, "", new_filter_tree_root);
diff --git a/src/override.cpp b/src/override.cpp
index d8934fab..42dbdb95 100644
--- a/src/override.cpp
+++ b/src/override.cpp
@@ -178,6 +178,13 @@ Option<bool> override_t::parse(const nlohmann::json& override_json, const std::s
override.replace_query = override_json["replace_query"].get<std::string>();
}
+ if (override_json.count("metadata") != 0) {
+ if(!override_json["metadata"].is_object()) {
+ return Option<bool>(400, "The `metadata` must be a JSON object.");
+ }
+ override.metadata = override_json["metadata"];
+ }
+
if(override_json.count("remove_matched_tokens") != 0) {
override.remove_matched_tokens = override_json["remove_matched_tokens"].get<bool>();
} else {
diff --git a/test/collection_override_test.cpp b/test/collection_override_test.cpp
index 73c3ec98..c529a729 100644
--- a/test/collection_override_test.cpp
+++ b/test/collection_override_test.cpp
@@ -121,6 +121,7 @@ TEST_F(CollectionOverrideTest, ExcludeIncludeExactQueryMatch) {
ASSERT_EQ(3, results["hits"].size());
ASSERT_EQ(3, results["found"].get<uint32_t>());
+ ASSERT_FALSE(results.contains("metadata"));
ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("3", results["hits"][1]["document"]["id"].get<std::string>().c_str());
@@ -308,7 +309,8 @@ TEST_F(CollectionOverrideTest, IncludeHitsFilterOverrides) {
{"query", "not-found"},
{"match", override_t::MATCH_EXACT}
}
- }
+ },
+ {"metadata", {{"foo", "bar"}}},
};
override_json_include["includes"] = nlohmann::json::array();
@@ -338,6 +340,7 @@ TEST_F(CollectionOverrideTest, IncludeHitsFilterOverrides) {
spp::sparse_hash_set<std::string>(), 10, "starring: will").get();
ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("bar", results["metadata"]["foo"].get<std::string>());
// disable filter curation option
override_json_include["filter_curated_hits"] = false;
@@ -2757,7 +2760,8 @@ TEST_F(CollectionOverrideTest, DynamicFilteringMultipleRuleMatch) {
},
{"remove_matched_tokens", true},
{"stop_processing", false},
- {"filter_by", "tags: twitter"}
+ {"filter_by", "tags: twitter"},
+ {"metadata", {{"foo", "bar"}}},
};
override_t override_filter1;
@@ -2790,6 +2794,7 @@ TEST_F(CollectionOverrideTest, DynamicFilteringMultipleRuleMatch) {
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("bar", results["metadata"]["foo"].get<std::string>());
// when stop_processing is enabled (default is true)
override_filter1_json.erase("stop_processing");
@@ -3577,7 +3582,8 @@ TEST_F(CollectionOverrideTest, OverrideWithTagsWithoutStopProcessing) {
},
"stop_processing": false,
"remove_matched_tokens": false,
- "filter_by": "category: kitchen"
+ "filter_by": "category: kitchen",
+ "metadata": {"foo": "bar"}
})"_json;
override_t override2;
@@ -3614,6 +3620,60 @@ TEST_F(CollectionOverrideTest, OverrideWithTagsWithoutStopProcessing) {
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("bar", results["metadata"]["foo"].get<std::string>());
+
+ collectionManager.drop_collection("coll1");
+}
+
+TEST_F(CollectionOverrideTest, MetadataValidation) {
+ Collection* coll1;
+
+ std::vector<field> fields = {field("name", field_types::STRING, false),
+ field("category", field_types::STRING_ARRAY, true),};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if (coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "").get();
+ }
+
+ nlohmann::json doc1;
+ doc1["id"] = "0";
+ doc1["name"] = "queryA";
+ doc1["category"] = {"kids"};
+
+ ASSERT_TRUE(coll1->add(doc1.dump()).ok());
+
+ std::vector<sort_by> sort_fields = {sort_by("_text_match", "DESC")};
+
+ nlohmann::json override_json1 = R"({
+ "id": "ov-1",
+ "rule": {
+ "query": "queryA",
+ "match": "exact"
+ },
+ "filter_by": "category: kids",
+ "metadata": "foo"
+ })"_json;
+
+ override_t override1;
+ auto op = override_t::parse(override_json1, "ov-1", override1);
+ ASSERT_FALSE(op.ok());
+ ASSERT_EQ("The `metadata` must be a JSON object.", op.error());
+
+ override_json1 = R"({
+ "id": "ov-1",
+ "rule": {
+ "query": "queryA",
+ "match": "exact"
+ },
+ "filter_by": "category: kids",
+ "metadata": {"foo": "bar"}
+ })"_json;
+
+ op = override_t::parse(override_json1, "ov-1", override1);
+ ASSERT_TRUE(op.ok());
+
+ coll1->add_override(override1);
collectionManager.drop_collection("coll1");
}