Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Parameterize `snippet_threshold` for highlighting.

kishorenc9622d5999621
diff --git a/include/collection.h b/include/collection.h
index 6711dcee..da3043ba 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -147,7 +147,7 @@ private:
void highlight_result(const field &search_field, const std::vector<std::vector<art_leaf *>> &searched_queries,
const KV &field_order_kv, const nlohmann::json &document,
- StringUtils & string_utils, highlight_t &highlight);
+ StringUtils & string_utils, size_t snippet_threshold, highlight_t &highlight);
void remove_document(nlohmann::json & document, const uint32_t seq_id, bool remove_from_store);
@@ -222,7 +222,8 @@ public:
const spp::sparse_hash_set<std::string> & include_fields = spp::sparse_hash_set<std::string>(),
const spp::sparse_hash_set<std::string> & exclude_fields = spp::sparse_hash_set<std::string>(),
size_t max_facet_values=10, size_t max_hits=500,
- const std::string & simple_facet_query = "");
+ const std::string & simple_facet_query = "",
+ const size_t snippet_threshold = 30);
Option<nlohmann::json> get(const std::string & id);
@@ -252,9 +253,6 @@ public:
const std::vector<Index *> &_get_indexes() const;
- // strings under this length will be fully highlighted, instead of showing a snippet of relevant portion
- enum {SNIPPET_STR_ABOVE_LEN = 30};
-
enum {MAX_ARRAY_MATCHES = 5};
// Using a $ prefix so that these meta keys stay above record entries in a lexicographically ordered KV store
diff --git a/src/collection.cpp b/src/collection.cpp
index 6c8bfe25..0b39d3e2 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -311,7 +311,8 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
const spp::sparse_hash_set<std::string> & include_fields,
const spp::sparse_hash_set<std::string> & exclude_fields,
const size_t max_facet_values, const size_t max_hits,
- const std::string & simple_facet_query) {
+ const std::string & simple_facet_query,
+ const size_t snippet_threshold) {
std::vector<uint32_t> included_ids;
std::vector<uint32_t> excluded_ids;
@@ -691,7 +692,8 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
if(query != "*" && (search_field.type == field_types::STRING ||
search_field.type == field_types::STRING_ARRAY)) {
highlight_t highlight;
- highlight_result(search_field, searched_queries, field_order_kv, document, string_utils, highlight);
+ highlight_result(search_field, searched_queries, field_order_kv, document,
+ string_utils, snippet_threshold, highlight);
if(!highlight.snippets.empty()) {
highlights.push_back(highlight);
}
@@ -913,7 +915,7 @@ void Collection::compute_facet_results(const facet &a_facet, double &fvmin, doub
void Collection::highlight_result(const field &search_field,
const std::vector<std::vector<art_leaf *>> &searched_queries,
const KV & field_order_kv, const nlohmann::json & document,
- StringUtils & string_utils, highlight_t & highlight) {
+ StringUtils & string_utils, size_t snippet_threshold, highlight_t & highlight) {
spp::sparse_hash_map<const art_leaf*, uint32_t*> leaf_to_indices;
std::vector<art_leaf *> query_suggestion;
@@ -992,11 +994,11 @@ void Collection::highlight_result(const field &search_field,
auto minmax = std::minmax_element(token_indices.begin(), token_indices.end());
- // For longer strings, pick surrounding tokens within N tokens of min_index and max_index for the snippet
- const size_t start_index = (tokens.size() <= SNIPPET_STR_ABOVE_LEN) ? 0 :
- std::max(0, (int)(*(minmax.first) - 5));
+ // For longer strings, pick surrounding tokens within 4 tokens of min_index and max_index for the snippet
+ const size_t start_index = (tokens.size() <= snippet_threshold) ? 0 :
+ std::max(0, (int)(*(minmax.first) - 4));
- const size_t end_index = (tokens.size() <= SNIPPET_STR_ABOVE_LEN) ? tokens.size() :
+ const size_t end_index = (tokens.size() <= snippet_threshold) ? tokens.size() :
std::min((int)tokens.size(), (int)(*(minmax.second) + 5));
std::stringstream snippet_stream;
diff --git a/src/core_api.cpp b/src/core_api.cpp
index 46e8b306..557bde91 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -185,6 +185,9 @@ void get_search(http_req & req, http_res & res) {
const char *INCLUDE_FIELDS = "include_fields";
const char *EXCLUDE_FIELDS = "exclude_fields";
+ // strings under this length will be fully highlighted, instead of showing a snippet of relevant portion
+ const char *SNIPPET_THRESHOLD = "snippet_threshold";
+
if(req.params.count(NUM_TYPOS) == 0) {
req.params[NUM_TYPOS] = "2";
}
@@ -222,6 +225,10 @@ void get_search(http_req & req, http_res & res) {
}
}
+ if(req.params.count(SNIPPET_THRESHOLD) == 0) {
+ req.params[SNIPPET_THRESHOLD] = "30";
+ }
+
if(req.params.count(PER_PAGE) == 0) {
req.params[PER_PAGE] = "10";
}
@@ -318,7 +325,8 @@ void get_search(http_req & req, http_res & res) {
include_fields, exclude_fields,
static_cast<size_t>(std::stoi(req.params[MAX_FACET_VALUES])),
static_cast<size_t>(std::stoi(req.params[MAX_HITS])),
- req.params[FACET_QUERY]);
+ req.params[FACET_QUERY],
+ static_cast<size_t>(std::stoi(req.params[SNIPPET_THRESHOLD])));
uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::high_resolution_clock::now() - begin).count();
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index e81087dd..89f234c0 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -2836,4 +2836,46 @@ TEST_F(CollectionTest, StringArrayFieldShouldNotAllowPlainString) {
ASSERT_STREQ("Field `categories` must be a string array.", add_op.error().c_str());
collectionManager.drop_collection("coll1");
-}
\ No newline at end of file
+}
+
+TEST_F(CollectionTest, SearchHighlightShouldFollowThreshold) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("title", field_types::STRING, true),
+ field("points", field_types::INT32, false)};
+
+ std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};
+
+ coll1 = collectionManager.get_collection("coll1");
+ if (coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", fields, "points").get();
+ }
+
+ nlohmann::json doc;
+ doc["id"] = "100";
+ doc["title"] = "The quick brown fox jumped over the lazy dog and ran straight to the forest to sleep.";
+ doc["points"] = 25;
+
+ auto add_op = coll1->add(doc.dump());
+ ASSERT_TRUE(add_op.ok());
+
+ // first with a large threshold
+
+ auto res = coll1->search("lazy", {"title"}, "", {}, sort_fields, 0, 10, 1,
+ token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, 500, "", 500).get();
+
+ ASSERT_STREQ("The quick brown fox jumped over the <mark>lazy</mark> dog and ran straight to the forest to sleep.",
+ res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
+
+ // now with with a small threshold (will show only 4 words either side of the matched token)
+
+ res = coll1->search("lazy", {"title"}, "", {}, sort_fields, 0, 10, 1,
+ token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, 500, "", 5).get();
+
+ ASSERT_STREQ("fox jumped over the <mark>lazy</mark> dog and ran straight",
+ res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
+
+ collectionManager.drop_collection("coll1");
+}