Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fixed a few higlighting/splitting edge cases.

kishorenc9533b73609ed
diff --git a/include/tokenizer.h b/include/tokenizer.h
index 4649d9b0..39c0b511 100644
--- a/include/tokenizer.h
+++ b/include/tokenizer.h
@@ -10,18 +10,26 @@ private:
size_t i;
const bool keep_empty;
const bool normalize;
+ const bool no_op;
size_t token_counter = 0;
iconv_t cd;
public:
- Tokenizer(const std::string& input, bool keep_empty=true, bool normalize=true):
- text(input), i(0), keep_empty(keep_empty), normalize(normalize) {
+ explicit Tokenizer(const std::string& input,
+ bool keep_empty=true, bool normalize=true, bool no_op=false):
+ text(input), i(0), keep_empty(keep_empty), normalize(normalize), no_op(no_op) {
cd = iconv_open("ASCII//TRANSLIT", "UTF-8");
}
+ ~Tokenizer() {
+ iconv_close(cd);
+ }
+
bool next(std::string& token, size_t& token_index);
void tokenize(std::vector<std::string>& tokens);
+
+ void tokenize(std::string& token);
};
\ No newline at end of file
diff --git a/src/collection.cpp b/src/collection.cpp
index c497271d..b58d94cd 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -10,6 +10,7 @@
#include <future>
#include <rocksdb/write_batch.h>
#include <system_metrics.h>
+#include <tokenizer.h>
#include "topster.h"
#include "logger.h"
@@ -1424,10 +1425,11 @@ void Collection::highlight_result(const field &search_field,
const Match & match = match_index.match;
std::vector<std::string> tokens;
+
if(search_field.type == field_types::STRING) {
- StringUtils::split(document[search_field.name], tokens, " ", true);
+ Tokenizer(document[search_field.name], true, false).tokenize(tokens);
} else {
- StringUtils::split(document[search_field.name][match_index.index], tokens, " ", true);
+ Tokenizer(document[search_field.name][match_index.index], true, false).tokenize(tokens);
}
std::vector<size_t> token_indices;
@@ -1442,7 +1444,8 @@ void Collection::highlight_result(const field &search_field,
continue;
}
std::string token = tokens[token_index];
- string_utils.unicode_normalize(token);
+ Tokenizer(token, true, true).tokenize(token);
+
token_hits.insert(token);
}
}
@@ -1470,7 +1473,7 @@ void Collection::highlight_result(const field &search_field,
}
std::string token = tokens[snippet_index];
- string_utils.unicode_normalize(token);
+ Tokenizer(token, true, true).tokenize(token);
if(token_hits.count(token) != 0) {
snippet_stream << highlight_start_tag << tokens[snippet_index] << highlight_end_tag;
@@ -1493,7 +1496,7 @@ void Collection::highlight_result(const field &search_field,
}
std::string token = tokens[value_index];
- string_utils.unicode_normalize(token);
+ Tokenizer(token, true, true).tokenize(token);
if(token_hits.count(token) != 0) {
value_stream << highlight_start_tag << tokens[value_index] << highlight_end_tag;
diff --git a/src/core_api.cpp b/src/core_api.cpp
index d0fa76aa..4551f24e 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -515,7 +515,9 @@ bool get_search(http_req & req, http_res & res) {
pinned_hits,
hidden_hits,
group_by_fields,
- static_cast<size_t>(std::stol(req.params[GROUP_LIMIT]))
+ static_cast<size_t>(std::stol(req.params[GROUP_LIMIT])),
+ req.params[HIGHLIGHT_START_TAG],
+ req.params[HIGHLIGHT_END_TAG]
);
uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(
diff --git a/src/index.cpp b/src/index.cpp
index 11eff26e..07000cb1 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -559,7 +559,7 @@ void Index::index_string_field(const std::string & text, const int64_t score, ar
uint32_t seq_id, int facet_id, const field & a_field) {
std::unordered_map<std::string, std::vector<uint32_t>> token_to_offsets;
- Tokenizer tokenizer(text, true, a_field.is_string());
+ Tokenizer tokenizer(text, true, true, !a_field.is_string());
std::string token;
size_t token_index = 0;
@@ -595,7 +595,7 @@ void Index::index_string_array_field(const std::vector<std::string> & strings, c
const std::string& str = strings[array_index];
std::set<std::string> token_set; // required to deal with repeating tokens
- Tokenizer tokenizer(str, true, a_field.is_string());
+ Tokenizer tokenizer(str, true, true, !a_field.is_string());
std::string token;
size_t token_index = 0;
@@ -744,7 +744,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
art_tree *t = search_index.at(facet_field.faceted_name());
std::vector<std::string> query_tokens;
- Tokenizer(facet_query.query, false, facet_field.is_string()).tokenize(query_tokens);
+ Tokenizer(facet_query.query, false, true, !facet_field.is_string()).tokenize(query_tokens);
for (size_t qtoken_index = 0; qtoken_index < query_tokens.size(); qtoken_index++) {
auto &q = query_tokens[qtoken_index];
@@ -1994,11 +1994,11 @@ void Index::tokenize_doc_field(const nlohmann::json& document, const field& sear
const std::string& field_name = search_field.name;
if(search_field.type == field_types::STRING) {
- Tokenizer(document[field_name], true, search_field.is_string()).tokenize(tokens);
+ Tokenizer(document[field_name], true, true, !search_field.is_string()).tokenize(tokens);
} else if(search_field.type == field_types::STRING_ARRAY) {
const std::vector<std::string>& values = document[field_name].get<std::vector<std::string>>();
for(const std::string & value: values) {
- Tokenizer(value, true, search_field.is_string()).tokenize(tokens);
+ Tokenizer(value, true, true, !search_field.is_string()).tokenize(tokens);
}
} else if(search_field.type == field_types::INT32) {
const int KEY_LEN = 8;
diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
index 713209d4..ec050be6 100644
--- a/src/tokenizer.cpp
+++ b/src/tokenizer.cpp
@@ -5,10 +5,16 @@ bool Tokenizer::next(std::string &token, size_t& token_index) {
std::stringstream out;
if(i >= text.size()) {
+ if(i == text.size() && !text.empty() && text.back() == ' ') {
+ token = "";
+ i++;
+ return true;
+ }
+
return false;
}
- if(!normalize) {
+ if(no_op) {
token = text;
i = text.size();
return true;
@@ -34,7 +40,9 @@ bool Tokenizer::next(std::string &token, size_t& token_index) {
return true;
}
- if(std::isalnum(text[i])) {
+ if(!normalize) {
+ out << text[i];
+ } else if(std::isalnum(text[i])) {
out << char(std::tolower(text[i]));
}
@@ -69,12 +77,17 @@ bool Tokenizer::next(std::string &token, size_t& token_index) {
} else {
// NOTE: outsize indicates bytes available AFTER current position so have to do <=
for(size_t out_index=0; out_index<5; out_index++) {
+ if(!normalize) {
+ out << outbuf[out_index];
+ continue;
+ }
+
bool is_ascii = ((outbuf[out_index] & ~0x7f) == 0);
bool keep_char = !is_ascii || std::isalnum(outbuf[out_index]);
if(keep_char) {
if(is_ascii && std::isalnum(outbuf[out_index])) {
- outbuf[out_index] = std::tolower(outbuf[out_index]);
+ outbuf[out_index] = char(std::tolower(outbuf[out_index]));
}
out << outbuf[out_index];
}
@@ -101,3 +114,8 @@ void Tokenizer::tokenize(std::vector<std::string> &tokens) {
tokens.push_back(token);
}
}
+
+void Tokenizer::tokenize(std::string& token) {
+ size_t token_index;
+ next(token, token_index);
+}
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 91d3fd8c..cbfc993a 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -785,7 +785,6 @@ TEST_F(CollectionTest, ArrayStringFieldHighlight) {
ASSERT_STREQ(results["hits"][0]["highlights"][1]["field"].get<std::string>().c_str(), "title");
ASSERT_STREQ(results["hits"][0]["highlights"][1]["snippet"].get<std::string>().c_str(),
"<mark>Amazing</mark> Spiderman is <mark>amazing</mark>"); // should highlight duplicating tokens
- LOG(INFO) << results["hits"][0]["highlights"][1];
ASSERT_EQ(2, results["hits"][0]["highlights"][1]["matched_tokens"].size());
ASSERT_STREQ("Amazing", results["hits"][0]["highlights"][1]["matched_tokens"][0].get<std::string>().c_str());
@@ -1985,33 +1984,41 @@ TEST_F(CollectionTest, SearchHighlightShouldUseHighlightTags) {
auto res = coll1->search("lazy", {"title"}, "", {}, sort_fields, 0, 10, 1,
token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,
- "<em>", "</em>").get();
+ "<em class=\"h\">", "</em>").get();
- ASSERT_STREQ("The quick brown fox jumped over the <em>lazy</em> fox. ",
+ ASSERT_STREQ("The quick brown fox jumped over the <em class=\"h\">lazy</em> fox. ",
res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
+}
- // empty tags
+TEST_F(CollectionTest, SearchHighlightWithNewLine) {
+ Collection *coll1;
- res = coll1->search("lazy", {"title"}, "", {}, sort_fields, 0, 10, 1,
- token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),
- spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,
- "", "").get();
+ std::vector<field> fields = {field("title", field_types::STRING, true),
+ field("points", field_types::INT32, false)};
- ASSERT_STREQ("The quick brown fox jumped over the lazy fox. ",
- res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
+ std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};
+
+ coll1 = collectionManager.get_collection("coll1");
+ if (coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
+ }
- // check for matched tokens
+ nlohmann::json doc;
+ doc["id"] = "100";
+ doc["title"] = "Blah, blah\nStark Industries";
+ doc["points"] = 25;
- res = coll1->search("fox", {"title"}, "", {}, sort_fields, 0, 10, 1,
- token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),
- spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0).get();
+ auto add_op = coll1->add(doc.dump());
+ ASSERT_TRUE(add_op.ok());
- ASSERT_EQ(2, res["hits"][0]["highlights"][0]["matched_tokens"].size());
- ASSERT_STREQ("fox",
- res["hits"][0]["highlights"][0]["matched_tokens"][0].get<std::string>().c_str());
+ auto res = coll1->search("stark", {"title"}, "", {}, sort_fields, 0, 10, 1,
+ token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0).get();
+
+ ASSERT_STREQ("Blah, blah <mark>Stark</mark> Industries",
+ res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
- ASSERT_STREQ("fox.",
- res["hits"][0]["highlights"][0]["matched_tokens"][1].get<std::string>().c_str());
+ ASSERT_STREQ("Stark", res["hits"][0]["highlights"][0]["matched_tokens"][0].get<std::string>().c_str());
collectionManager.drop_collection("coll1");
}
diff --git a/test/tokenizer_test.cpp b/test/tokenizer_test.cpp
index c53e5639..c9c0ea74 100644
--- a/test/tokenizer_test.cpp
+++ b/test/tokenizer_test.cpp
@@ -2,35 +2,37 @@
#include "tokenizer.h"
TEST(TokenizerTest, ShouldTokenizeNormalizeDifferentStrings) {
- const std::string withnewline = "Michael Jordan:\nWelcome, everybody. Welcome!";
+ const std::string withnewline = "Michael Jordan:\nWelcome, everybody. Welcome! ";
std::vector<std::string> tokens;
- Tokenizer(withnewline, true, true).tokenize(tokens);
- ASSERT_EQ(5, tokens.size());
+ Tokenizer(withnewline, true, true, false).tokenize(tokens);
+ ASSERT_EQ(6, tokens.size());
ASSERT_STREQ("michael", tokens[0].c_str());
ASSERT_STREQ("jordan", tokens[1].c_str());
ASSERT_STREQ("welcome", tokens[2].c_str());
ASSERT_STREQ("everybody", tokens[3].c_str());
ASSERT_STREQ("welcome", tokens[4].c_str());
+ ASSERT_STREQ("", tokens[5].c_str());
const std::string withspaces = " Michael Jordan ";
tokens.clear();
- Tokenizer(withspaces, true, true).tokenize(tokens);
- ASSERT_EQ(5, tokens.size());
+ Tokenizer(withspaces, true, true, false).tokenize(tokens);
+ ASSERT_EQ(6, tokens.size());
ASSERT_STREQ("", tokens[0].c_str());
ASSERT_STREQ("michael", tokens[1].c_str());
ASSERT_STREQ("", tokens[2].c_str());
ASSERT_STREQ("jordan", tokens[3].c_str());
ASSERT_STREQ("", tokens[4].c_str());
+ ASSERT_STREQ("", tokens[5].c_str());
tokens.clear();
- Tokenizer(withspaces, false, true).tokenize(tokens);
+ Tokenizer(withspaces, false, true, false).tokenize(tokens);
ASSERT_EQ(2, tokens.size());
ASSERT_STREQ("michael", tokens[0].c_str());
ASSERT_STREQ("jordan", tokens[1].c_str());
const std::string withspecialchars = "Special ½¥ and தமிழ் 你好吗 abcÅà123ß12 here.";
tokens.clear();
- Tokenizer(withspecialchars, false, true).tokenize(tokens);
+ Tokenizer(withspecialchars, false, true, false).tokenize(tokens);
ASSERT_EQ(7, tokens.size());
ASSERT_STREQ("special", tokens[0].c_str());
ASSERT_STREQ("12yen", tokens[1].c_str());
@@ -40,10 +42,10 @@ TEST(TokenizerTest, ShouldTokenizeNormalizeDifferentStrings) {
ASSERT_STREQ("abcaa123ss12", tokens[5].c_str());
ASSERT_STREQ("here", tokens[6].c_str());
- // when normalize is false, should be verbatim
+ // noop
tokens.clear();
- Tokenizer(withspecialchars, false, false).tokenize(tokens);
+ Tokenizer(withspecialchars, false, true, true).tokenize(tokens);
ASSERT_EQ(1, tokens.size());
ASSERT_STREQ(withspecialchars.c_str(), tokens[0].c_str());
}
@@ -51,7 +53,7 @@ TEST(TokenizerTest, ShouldTokenizeNormalizeDifferentStrings) {
TEST(TokenizerTest, ShouldTokenizeIteratively) {
const std::string withnewline = "Michael Jordan:\n\nWelcome, everybody. Welcome!";
std::vector<std::string> tokens;
- Tokenizer tokenizer1(withnewline, true, true);
+ Tokenizer tokenizer1(withnewline, true, true, false);
std::string token;
size_t token_index;
@@ -68,10 +70,10 @@ TEST(TokenizerTest, ShouldTokenizeIteratively) {
ASSERT_STREQ("everybody", tokens[4].c_str());
ASSERT_STREQ("welcome", tokens[5].c_str());
- // verbatim (normalize=false)
+ // verbatim (no_op=true)
tokens.clear();
- Tokenizer tokenizer2(withnewline, true, false);
+ Tokenizer tokenizer2(withnewline, true, false, true);
while(tokenizer2.next(token, token_index)) {
tokens.push_back(token);