Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Add response metadata via override.
Kishore Nallanc21c8557032a
diff --git a/include/collection.h b/include/collection.hindex bc949e61..9bfbc6f6 100644--- a/include/collection.h+++ b/include/collection.h@@ -194,7 +194,8 @@ private:std::vector<uint32_t>& excluded_ids,std::vector<const override_t*>& filter_overrides,bool& filter_curated_hits,- std::string& curated_sort_by) const;+ std::string& curated_sort_by,+ nlohmann::json& override_metadata) const;void curate_results(string& actual_query, const string& filter_query, bool enable_overrides, bool already_segmented,const std::set<std::string>& tags,@@ -203,7 +204,7 @@ private:std::vector<std::pair<uint32_t, uint32_t>>& included_ids,std::vector<uint32_t>& excluded_ids, std::vector<const override_t*>& filter_overrides,bool& filter_curated_hits,- std::string& curated_sort_by) const;+ std::string& curated_sort_by, nlohmann::json& override_metadata) const;static Option<bool> detect_new_fields(nlohmann::json& document,const DIRTY_VALUES& dirty_values,@@ -274,7 +275,8 @@ private:token_ordering token_order,filter_node_t*& filter_tree_root,std::vector<std::pair<uint32_t, uint32_t>>& included_ids,- std::vector<uint32_t>& excluded_ids) const;+ std::vector<uint32_t>& excluded_ids,+ nlohmann::json& override_metadata) const;void populate_text_match_info(nlohmann::json& info, uint64_t match_score, const text_match_type_t match_type) const;diff --git a/include/index.h b/include/index.hindex b9805daa..db881e87 100644--- a/include/index.h+++ b/include/index.h@@ -1022,7 +1022,8 @@ public:std::vector<std::string>& query_tokens,token_ordering token_order,filter_node_t*& filter_tree_root,- std::vector<const override_t*>& matched_dynamic_overrides) const;+ std::vector<const override_t*>& matched_dynamic_overrides,+ nlohmann::json& override_metadata) const;Option<bool> compute_sort_scores(const std::vector<sort_by>& sort_fields, const int* sort_order,std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3> field_values,diff --git a/include/override.h b/include/override.hindex 0496bc4b..c19a4639 100644--- a/include/override.h+++ b/include/override.h@@ -40,6 +40,8 @@ struct override_t {std::string sort_by;std::string replace_query;+ nlohmann::json metadata;+// epoch secondsint64_t effective_from_ts = -1;int64_t effective_to_ts = -1;diff --git a/src/collection.cpp b/src/collection.cppindex 7fc281b8..244f339f 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -821,7 +821,8 @@ bool Collection::does_override_match(const override_t& override, std::string& qustd::vector<uint32_t>& excluded_ids,std::vector<const override_t*>& filter_overrides,bool& filter_curated_hits,- std::string& curated_sort_by) const {+ std::string& curated_sort_by,+ nlohmann::json& override_metadata) const {auto now_epoch = int64_t(std::time(0));if(override.effective_from_ts != -1 && now_epoch < override.effective_from_ts) {@@ -888,6 +889,9 @@ bool Collection::does_override_match(const override_t& override, std::string& qufilter_curated_hits = override.filter_curated_hits;curated_sort_by = override.sort_by;+ if(override_metadata.empty()) {+ override_metadata = override.metadata;+ }return true;}@@ -900,7 +904,8 @@ void Collection::curate_results(string& actual_query, const string& filter_querystd::vector<uint32_t>& excluded_ids,std::vector<const override_t*>& filter_overrides,bool& filter_curated_hits,- std::string& curated_sort_by) const {+ std::string& curated_sort_by,+ nlohmann::json& override_metadata) const {std::set<uint32_t> excluded_set;@@ -951,7 +956,7 @@ void Collection::curate_results(string& actual_query, const string& filter_queryfilter_query, already_segmented, tags,pinned_hits, hidden_hits, included_ids,excluded_ids, filter_overrides, filter_curated_hits,- curated_sort_by);+ curated_sort_by, override_metadata);if(match_found) {all_tags_found = true;@@ -998,7 +1003,7 @@ void Collection::curate_results(string& actual_query, const string& filter_queryfilter_query, already_segmented, tags,pinned_hits, hidden_hits, included_ids,excluded_ids, filter_overrides, filter_curated_hits,- curated_sort_by);+ curated_sort_by, override_metadata);if(match_found) {found_overrides.insert(id);@@ -1014,7 +1019,8 @@ void Collection::curate_results(string& actual_query, const string& filter_queryconst auto& override = override_kv.second;bool match_found = does_override_match(override, query, excluded_set, actual_query, filter_query,already_segmented, tags, pinned_hits, hidden_hits, included_ids,- excluded_ids, filter_overrides, filter_curated_hits, curated_sort_by);+ excluded_ids, filter_overrides, filter_curated_hits,+ curated_sort_by, override_metadata);if(match_found && override.stop_processing) {break;}@@ -2042,6 +2048,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,std::vector<std::string> hidden_hits;StringUtils::split(hidden_hits_str, hidden_hits, ",");+ nlohmann::json override_metadata;std::vector<const override_t*> filter_overrides;bool filter_curated_hits = false;std::string curated_sort_by;@@ -2055,7 +2062,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,curate_results(query, filter_query, enable_overrides, pre_segmented_query, override_tag_set,pinned_hits, hidden_hits, included_ids, excluded_ids, filter_overrides, filter_curated_hits,- curated_sort_by);+ curated_sort_by, override_metadata);if(filter_curated_hits_option == 0 || filter_curated_hits_option == 1) {// When query param has explicit value set, override level configuration takes lower precedence.@@ -2152,7 +2159,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,// included_ids, excluded_idsprocess_filter_overrides(filter_overrides, q_include_tokens, token_order, filter_tree_root,- included_ids, excluded_ids);+ included_ids, excluded_ids, override_metadata);for(size_t i = 0; i < q_include_tokens.size(); i++) {auto& q_include_token = q_include_tokens[i];@@ -2888,6 +2895,10 @@ Option<nlohmann::json> Collection::search(std::string raw_query,result["request_params"]["per_page"] = per_page;result["request_params"]["q"] = raw_query;+ if(!override_metadata.empty()) {+ result["metadata"] = override_metadata;+ }+//long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();//!LOG(INFO) << "Time taken for result calc: " << timeMillis << "us";//!store->print_memory_usage();@@ -3161,11 +3172,12 @@ void Collection::process_filter_overrides(std::vector<const override_t*>& filtertoken_ordering token_order,filter_node_t*& filter_tree_root,std::vector<std::pair<uint32_t, uint32_t>>& included_ids,- std::vector<uint32_t>& excluded_ids) const {+ std::vector<uint32_t>& excluded_ids,+ nlohmann::json& override_metadata) const {std::vector<const override_t*> matched_dynamic_overrides;index->process_filter_overrides(filter_overrides, q_include_tokens, token_order,- filter_tree_root, matched_dynamic_overrides);+ filter_tree_root, matched_dynamic_overrides, override_metadata);// we will check the dynamic overrides to see if they also have include/excludestd::set<uint32_t> excluded_set;diff --git a/src/index.cpp b/src/index.cppindex 573c040c..cf0640d1 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -2130,7 +2130,8 @@ void Index::process_filter_overrides(const std::vector<const override_t*>& filtestd::vector<std::string>& query_tokens,token_ordering token_order,filter_node_t*& filter_tree_root,- std::vector<const override_t*>& matched_dynamic_overrides) const {+ std::vector<const override_t*>& matched_dynamic_overrides,+ nlohmann::json& override_metadata) const {std::shared_lock lock(mutex);for (auto& override : filter_overrides) {@@ -2140,6 +2141,9 @@ void Index::process_filter_overrides(const std::vector<const override_t*>& filtebool resolved_override = static_filter_query_eval(override, query_tokens, filter_tree_root);if (resolved_override) {+ if(override_metadata.empty()) {+ override_metadata = override->metadata;+ }if (override->remove_matched_tokens) {std::vector<std::string> rule_tokens;Tokenizer(override->rule.query, true).tokenize(rule_tokens);@@ -2166,6 +2170,10 @@ void Index::process_filter_overrides(const std::vector<const override_t*>& filtetoken_order, absorbed_tokens, filter_by_clause);if (resolved_override) {+ if(override_metadata.empty()) {+ override_metadata = override->metadata;+ }+filter_node_t* new_filter_tree_root = nullptr;Option<bool> filter_op = filter::parse_filter_query(filter_by_clause, search_schema,store, "", new_filter_tree_root);diff --git a/src/override.cpp b/src/override.cppindex d8934fab..42dbdb95 100644--- a/src/override.cpp+++ b/src/override.cpp@@ -178,6 +178,13 @@ Option<bool> override_t::parse(const nlohmann::json& override_json, const std::soverride.replace_query = override_json["replace_query"].get<std::string>();}+ if (override_json.count("metadata") != 0) {+ if(!override_json["metadata"].is_object()) {+ return Option<bool>(400, "The `metadata` must be a JSON object.");+ }+ override.metadata = override_json["metadata"];+ }+if(override_json.count("remove_matched_tokens") != 0) {override.remove_matched_tokens = override_json["remove_matched_tokens"].get<bool>();} else {diff --git a/test/collection_override_test.cpp b/test/collection_override_test.cppindex 73c3ec98..c529a729 100644--- a/test/collection_override_test.cpp+++ b/test/collection_override_test.cpp@@ -121,6 +121,7 @@ TEST_F(CollectionOverrideTest, ExcludeIncludeExactQueryMatch) {ASSERT_EQ(3, results["hits"].size());ASSERT_EQ(3, results["found"].get<uint32_t>());+ ASSERT_FALSE(results.contains("metadata"));ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());ASSERT_STREQ("3", results["hits"][1]["document"]["id"].get<std::string>().c_str());@@ -308,7 +309,8 @@ TEST_F(CollectionOverrideTest, IncludeHitsFilterOverrides) {{"query", "not-found"},{"match", override_t::MATCH_EXACT}}- }+ },+ {"metadata", {{"foo", "bar"}}},};override_json_include["includes"] = nlohmann::json::array();@@ -338,6 +340,7 @@ TEST_F(CollectionOverrideTest, IncludeHitsFilterOverrides) {spp::sparse_hash_set<std::string>(), 10, "starring: will").get();ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("bar", results["metadata"]["foo"].get<std::string>());// disable filter curation optionoverride_json_include["filter_curated_hits"] = false;@@ -2757,7 +2760,8 @@ TEST_F(CollectionOverrideTest, DynamicFilteringMultipleRuleMatch) {},{"remove_matched_tokens", true},{"stop_processing", false},- {"filter_by", "tags: twitter"}+ {"filter_by", "tags: twitter"},+ {"metadata", {{"foo", "bar"}}},};override_t override_filter1;@@ -2790,6 +2794,7 @@ TEST_F(CollectionOverrideTest, DynamicFilteringMultipleRuleMatch) {ASSERT_EQ(1, results["hits"].size());ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("bar", results["metadata"]["foo"].get<std::string>());// when stop_processing is enabled (default is true)override_filter1_json.erase("stop_processing");@@ -3577,7 +3582,8 @@ TEST_F(CollectionOverrideTest, OverrideWithTagsWithoutStopProcessing) {},"stop_processing": false,"remove_matched_tokens": false,- "filter_by": "category: kitchen"+ "filter_by": "category: kitchen",+ "metadata": {"foo": "bar"}})"_json;override_t override2;@@ -3614,6 +3620,60 @@ TEST_F(CollectionOverrideTest, OverrideWithTagsWithoutStopProcessing) {ASSERT_EQ(1, results["hits"].size());ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("bar", results["metadata"]["foo"].get<std::string>());++ collectionManager.drop_collection("coll1");+}++TEST_F(CollectionOverrideTest, MetadataValidation) {+ Collection* coll1;++ std::vector<field> fields = {field("name", field_types::STRING, false),+ field("category", field_types::STRING_ARRAY, true),};++ coll1 = collectionManager.get_collection("coll1").get();+ if (coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "").get();+ }++ nlohmann::json doc1;+ doc1["id"] = "0";+ doc1["name"] = "queryA";+ doc1["category"] = {"kids"};++ ASSERT_TRUE(coll1->add(doc1.dump()).ok());++ std::vector<sort_by> sort_fields = {sort_by("_text_match", "DESC")};++ nlohmann::json override_json1 = R"({+ "id": "ov-1",+ "rule": {+ "query": "queryA",+ "match": "exact"+ },+ "filter_by": "category: kids",+ "metadata": "foo"+ })"_json;++ override_t override1;+ auto op = override_t::parse(override_json1, "ov-1", override1);+ ASSERT_FALSE(op.ok());+ ASSERT_EQ("The `metadata` must be a JSON object.", op.error());++ override_json1 = R"({+ "id": "ov-1",+ "rule": {+ "query": "queryA",+ "match": "exact"+ },+ "filter_by": "category: kids",+ "metadata": {"foo": "bar"}+ })"_json;++ op = override_t::parse(override_json1, "ov-1", override1);+ ASSERT_TRUE(op.ok());++ coll1->add_override(override1);collectionManager.drop_collection("coll1");}