Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Parameterize `snippet_threshold` for highlighting.
kishorenc9622d5999621
diff --git a/include/collection.h b/include/collection.hindex 6711dcee..da3043ba 100644--- a/include/collection.h+++ b/include/collection.h@@ -147,7 +147,7 @@ private:void highlight_result(const field &search_field, const std::vector<std::vector<art_leaf *>> &searched_queries,const KV &field_order_kv, const nlohmann::json &document,- StringUtils & string_utils, highlight_t &highlight);+ StringUtils & string_utils, size_t snippet_threshold, highlight_t &highlight);void remove_document(nlohmann::json & document, const uint32_t seq_id, bool remove_from_store);@@ -222,7 +222,8 @@ public:const spp::sparse_hash_set<std::string> & include_fields = spp::sparse_hash_set<std::string>(),const spp::sparse_hash_set<std::string> & exclude_fields = spp::sparse_hash_set<std::string>(),size_t max_facet_values=10, size_t max_hits=500,- const std::string & simple_facet_query = "");+ const std::string & simple_facet_query = "",+ const size_t snippet_threshold = 30);Option<nlohmann::json> get(const std::string & id);@@ -252,9 +253,6 @@ public:const std::vector<Index *> &_get_indexes() const;- // strings under this length will be fully highlighted, instead of showing a snippet of relevant portion- enum {SNIPPET_STR_ABOVE_LEN = 30};-enum {MAX_ARRAY_MATCHES = 5};// Using a $ prefix so that these meta keys stay above record entries in a lexicographically ordered KV storediff --git a/src/collection.cpp b/src/collection.cppindex 6c8bfe25..0b39d3e2 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -311,7 +311,8 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::const spp::sparse_hash_set<std::string> & include_fields,const spp::sparse_hash_set<std::string> & exclude_fields,const size_t max_facet_values, const size_t max_hits,- const std::string & simple_facet_query) {+ const std::string & simple_facet_query,+ const size_t snippet_threshold) {std::vector<uint32_t> included_ids;std::vector<uint32_t> excluded_ids;@@ -691,7 +692,8 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::if(query != "*" && (search_field.type == field_types::STRING ||search_field.type == field_types::STRING_ARRAY)) {highlight_t highlight;- highlight_result(search_field, searched_queries, field_order_kv, document, string_utils, highlight);+ highlight_result(search_field, searched_queries, field_order_kv, document,+ string_utils, snippet_threshold, highlight);if(!highlight.snippets.empty()) {highlights.push_back(highlight);}@@ -913,7 +915,7 @@ void Collection::compute_facet_results(const facet &a_facet, double &fvmin, doubvoid Collection::highlight_result(const field &search_field,const std::vector<std::vector<art_leaf *>> &searched_queries,const KV & field_order_kv, const nlohmann::json & document,- StringUtils & string_utils, highlight_t & highlight) {+ StringUtils & string_utils, size_t snippet_threshold, highlight_t & highlight) {spp::sparse_hash_map<const art_leaf*, uint32_t*> leaf_to_indices;std::vector<art_leaf *> query_suggestion;@@ -992,11 +994,11 @@ void Collection::highlight_result(const field &search_field,auto minmax = std::minmax_element(token_indices.begin(), token_indices.end());- // For longer strings, pick surrounding tokens within N tokens of min_index and max_index for the snippet- const size_t start_index = (tokens.size() <= SNIPPET_STR_ABOVE_LEN) ? 0 :- std::max(0, (int)(*(minmax.first) - 5));+ // For longer strings, pick surrounding tokens within 4 tokens of min_index and max_index for the snippet+ const size_t start_index = (tokens.size() <= snippet_threshold) ? 0 :+ std::max(0, (int)(*(minmax.first) - 4));- const size_t end_index = (tokens.size() <= SNIPPET_STR_ABOVE_LEN) ? tokens.size() :+ const size_t end_index = (tokens.size() <= snippet_threshold) ? tokens.size() :std::min((int)tokens.size(), (int)(*(minmax.second) + 5));std::stringstream snippet_stream;diff --git a/src/core_api.cpp b/src/core_api.cppindex 46e8b306..557bde91 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -185,6 +185,9 @@ void get_search(http_req & req, http_res & res) {const char *INCLUDE_FIELDS = "include_fields";const char *EXCLUDE_FIELDS = "exclude_fields";+ // strings under this length will be fully highlighted, instead of showing a snippet of relevant portion+ const char *SNIPPET_THRESHOLD = "snippet_threshold";+if(req.params.count(NUM_TYPOS) == 0) {req.params[NUM_TYPOS] = "2";}@@ -222,6 +225,10 @@ void get_search(http_req & req, http_res & res) {}}+ if(req.params.count(SNIPPET_THRESHOLD) == 0) {+ req.params[SNIPPET_THRESHOLD] = "30";+ }+if(req.params.count(PER_PAGE) == 0) {req.params[PER_PAGE] = "10";}@@ -318,7 +325,8 @@ void get_search(http_req & req, http_res & res) {include_fields, exclude_fields,static_cast<size_t>(std::stoi(req.params[MAX_FACET_VALUES])),static_cast<size_t>(std::stoi(req.params[MAX_HITS])),- req.params[FACET_QUERY]);+ req.params[FACET_QUERY],+ static_cast<size_t>(std::stoi(req.params[SNIPPET_THRESHOLD])));uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();diff --git a/test/collection_test.cpp b/test/collection_test.cppindex e81087dd..89f234c0 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -2836,4 +2836,46 @@ TEST_F(CollectionTest, StringArrayFieldShouldNotAllowPlainString) {ASSERT_STREQ("Field `categories` must be a string array.", add_op.error().c_str());collectionManager.drop_collection("coll1");-}\ No newline at end of file+}++TEST_F(CollectionTest, SearchHighlightShouldFollowThreshold) {+ Collection *coll1;++ std::vector<field> fields = {field("title", field_types::STRING, true),+ field("points", field_types::INT32, false)};++ std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};++ coll1 = collectionManager.get_collection("coll1");+ if (coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", fields, "points").get();+ }++ nlohmann::json doc;+ doc["id"] = "100";+ doc["title"] = "The quick brown fox jumped over the lazy dog and ran straight to the forest to sleep.";+ doc["points"] = 25;++ auto add_op = coll1->add(doc.dump());+ ASSERT_TRUE(add_op.ok());++ // first with a large threshold++ auto res = coll1->search("lazy", {"title"}, "", {}, sort_fields, 0, 10, 1,+ token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, 500, "", 500).get();++ ASSERT_STREQ("The quick brown fox jumped over the <mark>lazy</mark> dog and ran straight to the forest to sleep.",+ res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());++ // now with with a small threshold (will show only 4 words either side of the matched token)++ res = coll1->search("lazy", {"title"}, "", {}, sort_fields, 0, 10, 1,+ token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, 500, "", 5).get();++ ASSERT_STREQ("fox jumped over the <mark>lazy</mark> dog and ran straight",+ res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());++ collectionManager.drop_collection("coll1");+}