Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fixed a few higlighting/splitting edge cases.
kishorenc9533b73609ed
diff --git a/include/tokenizer.h b/include/tokenizer.hindex 4649d9b0..39c0b511 100644--- a/include/tokenizer.h+++ b/include/tokenizer.h@@ -10,18 +10,26 @@ private:size_t i;const bool keep_empty;const bool normalize;+ const bool no_op;size_t token_counter = 0;iconv_t cd;public:- Tokenizer(const std::string& input, bool keep_empty=true, bool normalize=true):- text(input), i(0), keep_empty(keep_empty), normalize(normalize) {+ explicit Tokenizer(const std::string& input,+ bool keep_empty=true, bool normalize=true, bool no_op=false):+ text(input), i(0), keep_empty(keep_empty), normalize(normalize), no_op(no_op) {cd = iconv_open("ASCII//TRANSLIT", "UTF-8");}+ ~Tokenizer() {+ iconv_close(cd);+ }+bool next(std::string& token, size_t& token_index);void tokenize(std::vector<std::string>& tokens);++ void tokenize(std::string& token);};\ No newline at end of filediff --git a/src/collection.cpp b/src/collection.cppindex c497271d..b58d94cd 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -10,6 +10,7 @@#include <future>#include <rocksdb/write_batch.h>#include <system_metrics.h>+#include <tokenizer.h>#include "topster.h"#include "logger.h"@@ -1424,10 +1425,11 @@ void Collection::highlight_result(const field &search_field,const Match & match = match_index.match;std::vector<std::string> tokens;+if(search_field.type == field_types::STRING) {- StringUtils::split(document[search_field.name], tokens, " ", true);+ Tokenizer(document[search_field.name], true, false).tokenize(tokens);} else {- StringUtils::split(document[search_field.name][match_index.index], tokens, " ", true);+ Tokenizer(document[search_field.name][match_index.index], true, false).tokenize(tokens);}std::vector<size_t> token_indices;@@ -1442,7 +1444,8 @@ void Collection::highlight_result(const field &search_field,continue;}std::string token = tokens[token_index];- string_utils.unicode_normalize(token);+ Tokenizer(token, true, true).tokenize(token);+token_hits.insert(token);}}@@ -1470,7 +1473,7 @@ void Collection::highlight_result(const field &search_field,}std::string token = tokens[snippet_index];- string_utils.unicode_normalize(token);+ Tokenizer(token, true, true).tokenize(token);if(token_hits.count(token) != 0) {snippet_stream << highlight_start_tag << tokens[snippet_index] << highlight_end_tag;@@ -1493,7 +1496,7 @@ void Collection::highlight_result(const field &search_field,}std::string token = tokens[value_index];- string_utils.unicode_normalize(token);+ Tokenizer(token, true, true).tokenize(token);if(token_hits.count(token) != 0) {value_stream << highlight_start_tag << tokens[value_index] << highlight_end_tag;diff --git a/src/core_api.cpp b/src/core_api.cppindex d0fa76aa..4551f24e 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -515,7 +515,9 @@ bool get_search(http_req & req, http_res & res) {pinned_hits,hidden_hits,group_by_fields,- static_cast<size_t>(std::stol(req.params[GROUP_LIMIT]))+ static_cast<size_t>(std::stol(req.params[GROUP_LIMIT])),+ req.params[HIGHLIGHT_START_TAG],+ req.params[HIGHLIGHT_END_TAG]);uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(diff --git a/src/index.cpp b/src/index.cppindex 11eff26e..07000cb1 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -559,7 +559,7 @@ void Index::index_string_field(const std::string & text, const int64_t score, aruint32_t seq_id, int facet_id, const field & a_field) {std::unordered_map<std::string, std::vector<uint32_t>> token_to_offsets;- Tokenizer tokenizer(text, true, a_field.is_string());+ Tokenizer tokenizer(text, true, true, !a_field.is_string());std::string token;size_t token_index = 0;@@ -595,7 +595,7 @@ void Index::index_string_array_field(const std::vector<std::string> & strings, cconst std::string& str = strings[array_index];std::set<std::string> token_set; // required to deal with repeating tokens- Tokenizer tokenizer(str, true, a_field.is_string());+ Tokenizer tokenizer(str, true, true, !a_field.is_string());std::string token;size_t token_index = 0;@@ -744,7 +744,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,art_tree *t = search_index.at(facet_field.faceted_name());std::vector<std::string> query_tokens;- Tokenizer(facet_query.query, false, facet_field.is_string()).tokenize(query_tokens);+ Tokenizer(facet_query.query, false, true, !facet_field.is_string()).tokenize(query_tokens);for (size_t qtoken_index = 0; qtoken_index < query_tokens.size(); qtoken_index++) {auto &q = query_tokens[qtoken_index];@@ -1994,11 +1994,11 @@ void Index::tokenize_doc_field(const nlohmann::json& document, const field& searconst std::string& field_name = search_field.name;if(search_field.type == field_types::STRING) {- Tokenizer(document[field_name], true, search_field.is_string()).tokenize(tokens);+ Tokenizer(document[field_name], true, true, !search_field.is_string()).tokenize(tokens);} else if(search_field.type == field_types::STRING_ARRAY) {const std::vector<std::string>& values = document[field_name].get<std::vector<std::string>>();for(const std::string & value: values) {- Tokenizer(value, true, search_field.is_string()).tokenize(tokens);+ Tokenizer(value, true, true, !search_field.is_string()).tokenize(tokens);}} else if(search_field.type == field_types::INT32) {const int KEY_LEN = 8;diff --git a/src/tokenizer.cpp b/src/tokenizer.cppindex 713209d4..ec050be6 100644--- a/src/tokenizer.cpp+++ b/src/tokenizer.cpp@@ -5,10 +5,16 @@ bool Tokenizer::next(std::string &token, size_t& token_index) {std::stringstream out;if(i >= text.size()) {+ if(i == text.size() && !text.empty() && text.back() == ' ') {+ token = "";+ i++;+ return true;+ }+return false;}- if(!normalize) {+ if(no_op) {token = text;i = text.size();return true;@@ -34,7 +40,9 @@ bool Tokenizer::next(std::string &token, size_t& token_index) {return true;}- if(std::isalnum(text[i])) {+ if(!normalize) {+ out << text[i];+ } else if(std::isalnum(text[i])) {out << char(std::tolower(text[i]));}@@ -69,12 +77,17 @@ bool Tokenizer::next(std::string &token, size_t& token_index) {} else {// NOTE: outsize indicates bytes available AFTER current position so have to do <=for(size_t out_index=0; out_index<5; out_index++) {+ if(!normalize) {+ out << outbuf[out_index];+ continue;+ }+bool is_ascii = ((outbuf[out_index] & ~0x7f) == 0);bool keep_char = !is_ascii || std::isalnum(outbuf[out_index]);if(keep_char) {if(is_ascii && std::isalnum(outbuf[out_index])) {- outbuf[out_index] = std::tolower(outbuf[out_index]);+ outbuf[out_index] = char(std::tolower(outbuf[out_index]));}out << outbuf[out_index];}@@ -101,3 +114,8 @@ void Tokenizer::tokenize(std::vector<std::string> &tokens) {tokens.push_back(token);}}++void Tokenizer::tokenize(std::string& token) {+ size_t token_index;+ next(token, token_index);+}diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 91d3fd8c..cbfc993a 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -785,7 +785,6 @@ TEST_F(CollectionTest, ArrayStringFieldHighlight) {ASSERT_STREQ(results["hits"][0]["highlights"][1]["field"].get<std::string>().c_str(), "title");ASSERT_STREQ(results["hits"][0]["highlights"][1]["snippet"].get<std::string>().c_str(),"<mark>Amazing</mark> Spiderman is <mark>amazing</mark>"); // should highlight duplicating tokens- LOG(INFO) << results["hits"][0]["highlights"][1];ASSERT_EQ(2, results["hits"][0]["highlights"][1]["matched_tokens"].size());ASSERT_STREQ("Amazing", results["hits"][0]["highlights"][1]["matched_tokens"][0].get<std::string>().c_str());@@ -1985,33 +1984,41 @@ TEST_F(CollectionTest, SearchHighlightShouldUseHighlightTags) {auto res = coll1->search("lazy", {"title"}, "", {}, sort_fields, 0, 10, 1,token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,- "<em>", "</em>").get();+ "<em class=\"h\">", "</em>").get();- ASSERT_STREQ("The quick brown fox jumped over the <em>lazy</em> fox. ",+ ASSERT_STREQ("The quick brown fox jumped over the <em class=\"h\">lazy</em> fox. ",res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());+}- // empty tags+TEST_F(CollectionTest, SearchHighlightWithNewLine) {+ Collection *coll1;- res = coll1->search("lazy", {"title"}, "", {}, sort_fields, 0, 10, 1,- token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),- spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,- "", "").get();+ std::vector<field> fields = {field("title", field_types::STRING, true),+ field("points", field_types::INT32, false)};- ASSERT_STREQ("The quick brown fox jumped over the lazy fox. ",- res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());+ std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};++ coll1 = collectionManager.get_collection("coll1");+ if (coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();+ }- // check for matched tokens+ nlohmann::json doc;+ doc["id"] = "100";+ doc["title"] = "Blah, blah\nStark Industries";+ doc["points"] = 25;- res = coll1->search("fox", {"title"}, "", {}, sort_fields, 0, 10, 1,- token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),- spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0).get();+ auto add_op = coll1->add(doc.dump());+ ASSERT_TRUE(add_op.ok());- ASSERT_EQ(2, res["hits"][0]["highlights"][0]["matched_tokens"].size());- ASSERT_STREQ("fox",- res["hits"][0]["highlights"][0]["matched_tokens"][0].get<std::string>().c_str());+ auto res = coll1->search("stark", {"title"}, "", {}, sort_fields, 0, 10, 1,+ token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0).get();++ ASSERT_STREQ("Blah, blah <mark>Stark</mark> Industries",+ res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());- ASSERT_STREQ("fox.",- res["hits"][0]["highlights"][0]["matched_tokens"][1].get<std::string>().c_str());+ ASSERT_STREQ("Stark", res["hits"][0]["highlights"][0]["matched_tokens"][0].get<std::string>().c_str());collectionManager.drop_collection("coll1");}diff --git a/test/tokenizer_test.cpp b/test/tokenizer_test.cppindex c53e5639..c9c0ea74 100644--- a/test/tokenizer_test.cpp+++ b/test/tokenizer_test.cpp@@ -2,35 +2,37 @@#include "tokenizer.h"TEST(TokenizerTest, ShouldTokenizeNormalizeDifferentStrings) {- const std::string withnewline = "Michael Jordan:\nWelcome, everybody. Welcome!";+ const std::string withnewline = "Michael Jordan:\nWelcome, everybody. Welcome! ";std::vector<std::string> tokens;- Tokenizer(withnewline, true, true).tokenize(tokens);- ASSERT_EQ(5, tokens.size());+ Tokenizer(withnewline, true, true, false).tokenize(tokens);+ ASSERT_EQ(6, tokens.size());ASSERT_STREQ("michael", tokens[0].c_str());ASSERT_STREQ("jordan", tokens[1].c_str());ASSERT_STREQ("welcome", tokens[2].c_str());ASSERT_STREQ("everybody", tokens[3].c_str());ASSERT_STREQ("welcome", tokens[4].c_str());+ ASSERT_STREQ("", tokens[5].c_str());const std::string withspaces = " Michael Jordan ";tokens.clear();- Tokenizer(withspaces, true, true).tokenize(tokens);- ASSERT_EQ(5, tokens.size());+ Tokenizer(withspaces, true, true, false).tokenize(tokens);+ ASSERT_EQ(6, tokens.size());ASSERT_STREQ("", tokens[0].c_str());ASSERT_STREQ("michael", tokens[1].c_str());ASSERT_STREQ("", tokens[2].c_str());ASSERT_STREQ("jordan", tokens[3].c_str());ASSERT_STREQ("", tokens[4].c_str());+ ASSERT_STREQ("", tokens[5].c_str());tokens.clear();- Tokenizer(withspaces, false, true).tokenize(tokens);+ Tokenizer(withspaces, false, true, false).tokenize(tokens);ASSERT_EQ(2, tokens.size());ASSERT_STREQ("michael", tokens[0].c_str());ASSERT_STREQ("jordan", tokens[1].c_str());const std::string withspecialchars = "Special ½¥ and தமிழ் 你好吗 abcÅà123ß12 here.";tokens.clear();- Tokenizer(withspecialchars, false, true).tokenize(tokens);+ Tokenizer(withspecialchars, false, true, false).tokenize(tokens);ASSERT_EQ(7, tokens.size());ASSERT_STREQ("special", tokens[0].c_str());ASSERT_STREQ("12yen", tokens[1].c_str());@@ -40,10 +42,10 @@ TEST(TokenizerTest, ShouldTokenizeNormalizeDifferentStrings) {ASSERT_STREQ("abcaa123ss12", tokens[5].c_str());ASSERT_STREQ("here", tokens[6].c_str());- // when normalize is false, should be verbatim+ // nooptokens.clear();- Tokenizer(withspecialchars, false, false).tokenize(tokens);+ Tokenizer(withspecialchars, false, true, true).tokenize(tokens);ASSERT_EQ(1, tokens.size());ASSERT_STREQ(withspecialchars.c_str(), tokens[0].c_str());}@@ -51,7 +53,7 @@ TEST(TokenizerTest, ShouldTokenizeNormalizeDifferentStrings) {TEST(TokenizerTest, ShouldTokenizeIteratively) {const std::string withnewline = "Michael Jordan:\n\nWelcome, everybody. Welcome!";std::vector<std::string> tokens;- Tokenizer tokenizer1(withnewline, true, true);+ Tokenizer tokenizer1(withnewline, true, true, false);std::string token;size_t token_index;@@ -68,10 +70,10 @@ TEST(TokenizerTest, ShouldTokenizeIteratively) {ASSERT_STREQ("everybody", tokens[4].c_str());ASSERT_STREQ("welcome", tokens[5].c_str());- // verbatim (normalize=false)+ // verbatim (no_op=true)tokens.clear();- Tokenizer tokenizer2(withnewline, true, false);+ Tokenizer tokenizer2(withnewline, true, false, true);while(tokenizer2.next(token, token_index)) {tokens.push_back(token);