Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Make symbols for indexing and segmentation configurable.
Kishore Nallan07d838e38543
diff --git a/include/collection.h b/include/collection.hindex 4d3d5c74..065038bf 100644--- a/include/collection.h+++ b/include/collection.h@@ -325,6 +325,10 @@ private:std::vector<field> dynamic_fields;+ std::vector<char> symbols_to_index;++ std::vector<char> token_separators;+const std::vector<Index*> indices;// methods@@ -389,6 +393,8 @@ private:std::vector<Index *> init_indices();+ static std::vector<char> to_char_array(std::vector<std::string> strs);+public:enum {MAX_ARRAY_MATCHES = 5};@@ -413,10 +419,8 @@ public:static constexpr const char* COLLECTION_NUM_MEMORY_SHARDS = "num_memory_shards";static constexpr const char* COLLECTION_FALLBACK_FIELD_TYPE = "fallback_field_type";- // DON'T CHANGE THESE VALUES!- // this key is used as namespace key to store metadata about the document- static constexpr const char* DOC_META_KEY = "$TSM$_";- static constexpr const char* DOC_META_DIRTY_VALUES_KEY = "dirty_values";+ static constexpr const char* COLLECTION_SYMBOLS_TO_INDEX = "symbols_to_index";+ static constexpr const char* COLLECTION_SEPARATORS = "token_separators";// methods@@ -425,7 +429,8 @@ public:Collection(const std::string& name, const uint32_t collection_id, const uint64_t created_at,const uint32_t next_seq_id, Store *store, const std::vector<field>& fields,const std::string& default_sorting_field, const size_t num_memory_shards,- const float max_memory_ratio, const std::string& fallback_field_type);+ const float max_memory_ratio, const std::string& fallback_field_type,+ const std::vector<std::string>& symbols_to_index, const std::vector<std::string>& token_separators);~Collection();@@ -556,6 +561,10 @@ public:DIRTY_VALUES parse_dirty_values_option(std::string& dirty_values) const;+ std::vector<char> get_symbols_to_index();++ std::vector<char> get_token_separators();+// Override operationsOption<uint32_t> add_override(const override_t & override);diff --git a/include/collection_manager.h b/include/collection_manager.hindex 58261887..8027755c 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -134,7 +134,9 @@ public:const std::vector<field> & fields,const std::string & default_sorting_field="",const uint64_t created_at = static_cast<uint64_t>(std::time(nullptr)),- const std::string& fallback_field_type = "");+ const std::string& fallback_field_type = "",+ const std::vector<std::string>& symbols_to_index = {},+ const std::vector<std::string>& token_separators = {});locked_resource_view_t<Collection> get_collection(const std::string & collection_name) const;diff --git a/include/index.h b/include/index.hindex 39dfe6aa..77f447cd 100644--- a/include/index.h+++ b/include/index.h@@ -189,6 +189,10 @@ private:// this is used for wildcard queriessorted_array seq_ids;+ std::vector<char> symbols_to_index;++ std::vector<char> token_separators;+StringUtils string_utils;// used as sentinels@@ -312,7 +316,8 @@ public:Index() = delete;Index(const std::string name, const std::unordered_map<std::string, field> & search_schema,- std::map<std::string, field> facet_schema, std::unordered_map<std::string, field> sort_schema);+ std::map<std::string, field> facet_schema, std::unordered_map<std::string, field> sort_schema,+ const std::vector<char>& symbols_to_index, const std::vector<char>& token_separators);~Index();diff --git a/include/tokenizer.h b/include/tokenizer.hindex 04b7ef4e..ba9e101f 100644--- a/include/tokenizer.h+++ b/include/tokenizer.h@@ -24,6 +24,7 @@ private:static const size_t SKIP = 2;uint8_t index_symbols[256] = {};+ uint8_t separator_symbols[256] = {};std::string out;@@ -42,7 +43,7 @@ private:inline size_t get_stream_mode(char c) {return (std::isalnum(c) || index_symbols[uint8_t(c)] == 1) ? INDEX : (- (c == ' ' || c == '\n') ? SEPARATE : SKIP+ (c == ' ' || c == '\n' || separator_symbols[uint8_t(c)] == 1) ? SEPARATE : SKIP);}@@ -55,7 +56,8 @@ public:explicit Tokenizer(const std::string& input,bool normalize=true, bool no_op=false,const std::string& locale = "",- const std::vector<char>& symbols_to_index = {});+ const std::vector<char>& symbols_to_index = {},+ const std::vector<char>& separators = {});~Tokenizer() {iconv_close(cd);diff --git a/src/collection.cpp b/src/collection.cppindex 0f5564e6..b1017d16 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -44,13 +44,15 @@ struct match_index_t {Collection::Collection(const std::string& name, const uint32_t collection_id, const uint64_t created_at,const uint32_t next_seq_id, Store *store, const std::vector<field> &fields,const std::string& default_sorting_field, const size_t num_memory_shards,- const float max_memory_ratio, const std::string& fallback_field_type):+ const float max_memory_ratio, const std::string& fallback_field_type,+ const std::vector<std::string>& symbols_to_index, const std::vector<std::string>& token_separators):name(name), collection_id(collection_id), created_at(created_at),next_seq_id(next_seq_id), store(store),fields(fields), default_sorting_field(default_sorting_field),num_memory_shards(num_memory_shards),max_memory_ratio(max_memory_ratio),fallback_field_type(fallback_field_type), dynamic_fields({}),+ symbols_to_index(to_char_array(symbols_to_index)), token_separators(to_char_array(token_separators)),indices(init_indices()) {this->num_documents = 0;@@ -1369,7 +1371,10 @@ void Collection::parse_search_query(const std::string &query, std::vector<std::sif(already_segmented) {StringUtils::split(query, tokens, " ");} else {- Tokenizer(query, true, false, locale, {'-'}).tokenize(tokens);+ std::vector<char> custom_symbols = symbols_to_index;+ custom_symbols.push_back('-');++ Tokenizer(query, true, false, locale, custom_symbols).tokenize(tokens);}bool exclude_operator_prior = false;@@ -1378,17 +1383,24 @@ void Collection::parse_search_query(const std::string &query, std::vector<std::sif(token[0] == '-') {exclude_operator_prior = true;token = token.substr(1);- } else {- // remove hyphens (added earlier for supporting negation syntax)- // to maintain parity with indexing which currently removing special symbols- token.erase(std::remove(token.begin(), token.end(), '-'), token.end());}- if(exclude_operator_prior) {- q_exclude_tokens.push_back(token);- exclude_operator_prior = false;+ // retokenize using collection config+ std::vector<std::string> sub_tokens;++ if(already_segmented) {+ StringUtils::split(token, sub_tokens, " ");} else {- q_include_tokens.push_back(token);+ Tokenizer(token, true, false, locale, symbols_to_index, token_separators).tokenize(sub_tokens);+ }++ for(auto& sub_token: sub_tokens) {+ if(exclude_operator_prior) {+ q_exclude_tokens.push_back(sub_token);+ exclude_operator_prior = false;+ } else {+ q_include_tokens.push_back(sub_token);+ }}}@@ -1673,7 +1685,7 @@ void Collection::highlight_result(const field &search_field,text = document[search_field.name][match_index.index];}- Tokenizer tokenizer(text, true, false, search_field.locale);+ Tokenizer tokenizer(text, true, false, search_field.locale, symbols_to_index, token_separators);if(search_field.locale == "ko") {text = string_utils.unicode_nfkd(text);@@ -2777,7 +2789,8 @@ std::vector<Index *> Collection::init_indices() {}for(size_t i = 0; i < num_memory_shards; i++) {- Index* index = new Index(name+std::to_string(i), search_schema, facet_schema, sort_schema);+ Index* index = new Index(name+std::to_string(i), search_schema, facet_schema, sort_schema,+ symbols_to_index, token_separators);index_list.push_back(index);}@@ -2800,3 +2813,22 @@ DIRTY_VALUES Collection::parse_dirty_values_option(std::string& dirty_values) coreturn dirty_values_action;}++std::vector<char> Collection::to_char_array(std::vector<std::string> strings) {+ std::vector<char> vec;+ for(const auto& s: strings) {+ if(s.length() == 1) {+ vec.push_back(s[0]);+ }+ }++ return vec;+}++std::vector<char> Collection::get_symbols_to_index() {+ return symbols_to_index;+}++std::vector<char> Collection::get_token_separators() {+ return token_separators;+}diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex de6ed66a..b11edbf3 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -52,6 +52,17 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collectioncollection_meta[Collection::COLLECTION_FALLBACK_FIELD_TYPE].get<std::string>() :"";+ std::vector<std::string> symbols_to_index;+ std::vector<std::string> token_separators;++ if(collection_meta.count(Collection::COLLECTION_SYMBOLS_TO_INDEX) != 0) {+ symbols_to_index = collection_meta[Collection::COLLECTION_SYMBOLS_TO_INDEX].get<std::vector<std::string>>();+ }++ if(collection_meta.count(Collection::COLLECTION_SEPARATORS) != 0) {+ token_separators = collection_meta[Collection::COLLECTION_SEPARATORS].get<std::vector<std::string>>();+ }+LOG(INFO) << "Found collection " << this_collection_name << " with " << num_memory_shards << " memory shards.";Collection* collection = new Collection(this_collection_name,@@ -63,7 +74,9 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collectiondefault_sorting_field,num_memory_shards,max_memory_ratio,- fallback_field_type);+ fallback_field_type,+ symbols_to_index,+ token_separators);return collection;}@@ -222,7 +235,9 @@ Option<Collection*> CollectionManager::create_collection(const std::string& nameconst std::vector<field> & fields,const std::string& default_sorting_field,const uint64_t created_at,- const std::string& fallback_field_type) {+ const std::string& fallback_field_type,+ const std::vector<std::string>& symbols_to_index,+ const std::vector<std::string>& token_separators) {if(store->contains(Collection::get_meta_key(name))) {return Option<Collection*>(409, std::string("A collection with name `") + name + "` already exists.");@@ -252,10 +267,13 @@ Option<Collection*> CollectionManager::create_collection(const std::string& namecollection_meta[Collection::COLLECTION_CREATED] = created_at;collection_meta[Collection::COLLECTION_NUM_MEMORY_SHARDS] = num_memory_shards;collection_meta[Collection::COLLECTION_FALLBACK_FIELD_TYPE] = fallback_field_type;+ collection_meta[Collection::COLLECTION_SYMBOLS_TO_INDEX] = symbols_to_index;+ collection_meta[Collection::COLLECTION_SEPARATORS] = token_separators;Collection* new_collection = new Collection(name, next_collection_id, created_at, 0, store, fields,default_sorting_field, num_memory_shards,- this->max_memory_ratio, fallback_field_type);+ this->max_memory_ratio, fallback_field_type,+ symbols_to_index, token_separators);next_collection_id++;rocksdb::WriteBatch batch;@@ -838,6 +856,8 @@ nlohmann::json CollectionManager::get_collection_summaries() const {Option<Collection*> CollectionManager::create_collection(nlohmann::json& req_json) {const char* NUM_MEMORY_SHARDS = "num_memory_shards";+ const char* SYMBOLS_TO_INDEX = "symbols_to_index";+ const char* TOKEN_SEPARATORS = "token_separators";const char* DEFAULT_SORTING_FIELD = "default_sorting_field";// validate presence of mandatory fields@@ -854,6 +874,14 @@ Option<Collection*> CollectionManager::create_collection(nlohmann::json& req_jsoreq_json[NUM_MEMORY_SHARDS] = CollectionManager::DEFAULT_NUM_MEMORY_SHARDS;}+ if(req_json.count(SYMBOLS_TO_INDEX) == 0) {+ req_json[SYMBOLS_TO_INDEX] = std::vector<std::string>();+ }++ if(req_json.count(TOKEN_SEPARATORS) == 0) {+ req_json[TOKEN_SEPARATORS] = std::vector<std::string>();+ }+if(req_json.count("fields") == 0) {return Option<Collection*>(400, "Parameter `fields` is required.");}@@ -871,6 +899,26 @@ Option<Collection*> CollectionManager::create_collection(nlohmann::json& req_jsoreturn Option<Collection*>(400, std::string("`") + NUM_MEMORY_SHARDS + "` should be a positive integer.");}+ if(!req_json[SYMBOLS_TO_INDEX].is_array()) {+ return Option<Collection*>(400, std::string("`") + SYMBOLS_TO_INDEX + "` should be an array of character symbols.");+ }++ if(!req_json[TOKEN_SEPARATORS].is_array()) {+ return Option<Collection*>(400, std::string("`") + TOKEN_SEPARATORS + "` should be an array of character symbols.");+ }++ for (auto it = req_json[SYMBOLS_TO_INDEX].begin(); it != req_json[SYMBOLS_TO_INDEX].end(); ++it) {+ if(!it->is_string() || it->get<std::string>().size() != 1 ) {+ return Option<Collection*>(400, std::string("`") + SYMBOLS_TO_INDEX + "` should be an array of character symbols.");+ }+ }++ for (auto it = req_json[TOKEN_SEPARATORS].begin(); it != req_json[TOKEN_SEPARATORS].end(); ++it) {+ if(!it->is_string() || it->get<std::string>().size() != 1 ) {+ return Option<Collection*>(400, std::string("`") + TOKEN_SEPARATORS + "` should be an array of character symbols.");+ }+ }+size_t num_memory_shards = req_json[NUM_MEMORY_SHARDS].get<size_t>();if(num_memory_shards == 0) {return Option<Collection*>(400, std::string("`") + NUM_MEMORY_SHARDS + "` should be a positive integer.");@@ -897,7 +945,9 @@ Option<Collection*> CollectionManager::create_collection(nlohmann::json& req_jsoreturn CollectionManager::get_instance().create_collection(req_json["name"], num_memory_shards,fields, default_sorting_field, created_at,- fallback_field_type);+ fallback_field_type,+ req_json[SYMBOLS_TO_INDEX],+ req_json[TOKEN_SEPARATORS]);}Option<bool> CollectionManager::load_collection(const nlohmann::json &collection_meta,diff --git a/src/index.cpp b/src/index.cppindex 9fb2c182..467ff762 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -24,8 +24,10 @@ spp::sparse_hash_map<uint32_t, int64_t> Index::text_match_sentinel_value;spp::sparse_hash_map<uint32_t, int64_t> Index::seq_id_sentinel_value;Index::Index(const std::string name, const std::unordered_map<std::string, field> & search_schema,- std::map<std::string, field> facet_schema, std::unordered_map<std::string, field> sort_schema):- name(name), search_schema(search_schema), facet_schema(facet_schema), sort_schema(sort_schema) {+ std::map<std::string, field> facet_schema, std::unordered_map<std::string, field> sort_schema,+ const std::vector<char>& symbols_to_index, const std::vector<char>& token_separators):+ name(name), search_schema(search_schema), facet_schema(facet_schema), sort_schema(sort_schema),+ symbols_to_index(symbols_to_index), token_separators(token_separators) {for(const auto & fname_field: search_schema) {if(fname_field.second.is_string()) {@@ -542,7 +544,7 @@ void Index::index_string_field(const std::string & text, const int64_t score, aruint32_t seq_id, bool is_facet, const field & a_field) {std::unordered_map<std::string, std::vector<uint32_t>> token_to_offsets;- Tokenizer tokenizer(text, true, !a_field.is_string(), a_field.locale);+ Tokenizer tokenizer(text, true, !a_field.is_string(), a_field.locale, symbols_to_index, token_separators);std::string token, last_token;size_t token_index = 0;@@ -597,7 +599,7 @@ void Index::index_string_array_field(const std::vector<std::string> & strings, cconst std::string& str = strings[array_index];std::set<std::string> token_set; // required to deal with repeating tokens- Tokenizer tokenizer(str, true, !a_field.is_string(), a_field.locale);+ Tokenizer tokenizer(str, true, !a_field.is_string(), a_field.locale, symbols_to_index, token_separators);std::string token, last_token;size_t token_index = 0;diff --git a/src/tokenizer.cpp b/src/tokenizer.cppindex 3ce7b7bd..1d868e01 100644--- a/src/tokenizer.cpp+++ b/src/tokenizer.cpp@@ -3,7 +3,8 @@#include "tokenizer.h"Tokenizer::Tokenizer(const std::string& input, bool normalize, bool no_op, const std::string& locale,- const std::vector<char>& symbols_to_index):+ const std::vector<char>& symbols_to_index,+ const std::vector<char>& separators):i(0), normalize(normalize), no_op(no_op), locale(locale) {if(locale == "zh") {@@ -50,6 +51,10 @@ Tokenizer::Tokenizer(const std::string& input, bool normalize, bool no_op, constindex_symbols[uint8_t(c)] = 1;}+ for(char c: separators) {+ separator_symbols[uint8_t(c)] = 1;+ }+UErrorCode errcode = U_ZERO_ERROR;nfkd = icu::Normalizer2::getNFKDInstance(errcode);}diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex ac87bfc4..edc11c58 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -33,7 +33,8 @@ protected:};sort_fields = { sort_by("points", "DESC") };- collection1 = collectionManager.create_collection("collection1", 4, search_fields, "points", 12345).get();+ collection1 = collectionManager.create_collection("collection1", 4, search_fields,+ "points", 12345, "", {"+"}, {"-"}).get();}virtual void SetUp() {@@ -94,7 +95,7 @@ TEST_F(CollectionManagerTest, CollectionCreation) {"{\"facet\":true,\"locale\":\"\",\"name\":\".*_year\",\"optional\":true,\"type\":\"int32\"},""{\"facet\":false,\"locale\":\"\",\"name\":\"location\",\"optional\":true,\"type\":\"geopoint\"},""{\"facet\":false,\"locale\":\"\",\"name\":\"points\",\"optional\":false,\"type\":\"int32\"}],\"id\":0,"- "\"name\":\"collection1\",\"num_memory_shards\":4}",+ "\"name\":\"collection1\",\"num_memory_shards\":4,\"symbols_to_index\":[\"+\"],\"token_separators\":[\"-\"]}",collection_meta_json);ASSERT_EQ("1", next_collection_id);}@@ -113,16 +114,27 @@ TEST_F(CollectionManagerTest, ShouldInitCollection) {delete collection;- // with created_at ts+ // with non-default valuesnlohmann::json collection_meta2 =nlohmann::json::parse("{\"name\": \"foobar\", \"id\": 100, \"fields\": [{\"name\": \"org\", \"type\": ""\"string\", \"facet\": false}], \"created_at\": 12345,"- "\"default_sorting_field\": \"foo\"}");+ "\"default_sorting_field\": \"foo\","+ "\"symbols_to_index\": [\"+\"], \"token_separators\": [\"-\"]}");+collection = collectionManager.init_collection(collection_meta2, 100, store, 1.0f);ASSERT_EQ(12345, collection->get_created_at());+ std::vector<char> expected_symbols = {'+'};+ std::vector<char> expected_separators = {'-'};++ ASSERT_EQ(1, collection->get_token_separators().size());+ ASSERT_EQ('-', collection->get_token_separators()[0]);++ ASSERT_EQ(1, collection->get_symbols_to_index().size());+ ASSERT_EQ('+', collection->get_symbols_to_index()[0]);+delete collection;}@@ -282,6 +294,15 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {ASSERT_EQ(0, synonyms.at("id3").root.size());ASSERT_EQ(2, synonyms.at("id3").synonyms.size());+ std::vector<char> expected_symbols = {'+'};+ std::vector<char> expected_separators = {'-'};++ ASSERT_EQ(1, collection1->get_token_separators().size());+ ASSERT_EQ('-', collection1->get_token_separators()[0]);++ ASSERT_EQ(1, collection1->get_symbols_to_index().size());+ ASSERT_EQ('+', collection1->get_symbols_to_index()[0]);+results = collection1->search("thomas", search_fields, "", facets, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();ASSERT_EQ(4, results["hits"].size());}diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cppindex 55ab32a4..3f7e291a 100644--- a/test/collection_specific_test.cpp+++ b/test/collection_specific_test.cpp@@ -952,4 +952,144 @@ TEST_F(CollectionSpecificTest, HighlightEmptyArray) {ASSERT_EQ("name", results["hits"][0]["highlights"][0]["field"]);collectionManager.drop_collection("coll1");-}\ No newline at end of file+}++TEST_F(CollectionSpecificTest, CustomSeparators) {+ std::vector<field> fields = {field("name", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ Collection* coll1 = collectionManager.create_collection(+ "coll1", 1, fields, "points", 0, "", {}, {"-"}+ ).get();++ nlohmann::json doc1;+ doc1["id"] = "0";+ doc1["name"] = "alpha-beta-gamma-omega-zeta";+ doc1["points"] = 100;++ ASSERT_TRUE(coll1->add(doc1.dump()).ok());++ auto results = coll1->search("gamma", {"name"},+ "", {}, {}, {0}, 10,+ 1, FREQUENCY, {false},+ 1, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1, {}, {}, {}, 0,+ "<mark>", "</mark>",{}, 1000,+ true, false, true, "", 1000).get();++ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ(1, results["hits"][0]["highlights"].size());+ ASSERT_EQ("name", results["hits"][0]["highlights"][0]["field"]);+ ASSERT_EQ("alpha-beta-<mark>gamma</mark>-omega-zeta", results["hits"][0]["highlights"][0]["snippet"]);++ // ensure that symbols are validated++ nlohmann::json coll_def;+ coll_def["fields"] = {+ {{"name", "foo"}, {"type", "string"}, {"facet", false}}+ };+ coll_def["name"] = "foo";+ coll_def["token_separators"] = {"foo"};++ auto coll_op = collectionManager.create_collection(coll_def);++ ASSERT_FALSE(coll_op.ok());+ ASSERT_EQ("`token_separators` should be an array of character symbols.", coll_op.error());++ coll_def["token_separators"] = "f";+ coll_op = collectionManager.create_collection(coll_def);++ ASSERT_FALSE(coll_op.ok());+ ASSERT_EQ("`token_separators` should be an array of character symbols.", coll_op.error());++ coll_def["token_separators"] = 123;+ coll_op = collectionManager.create_collection(coll_def);++ ASSERT_FALSE(coll_op.ok());+ ASSERT_EQ("`token_separators` should be an array of character symbols.", coll_op.error());++ collectionManager.drop_collection("coll1");+}++TEST_F(CollectionSpecificTest, CustomSymbolsForIndexing) {+ std::vector<field> fields = {field("name", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ Collection* coll1 = collectionManager.create_collection(+ "coll1", 1, fields, "points", 0, "", {"+"}, {}+ ).get();++ nlohmann::json doc1;+ doc1["id"] = "0";+ doc1["name"] = "Yes, C++ is great!";+ doc1["points"] = 100;++ nlohmann::json doc2;+ doc2["id"] = "1";+ doc2["name"] = "Yes, C is great!";+ doc2["points"] = 100;++ ASSERT_TRUE(coll1->add(doc1.dump()).ok());+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());++ auto results = coll1->search("c++", {"name"},+ "", {}, {}, {0}, 10,+ 1, FREQUENCY, {false},+ 1, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1, {}, {}, {}, 0,+ "<mark>", "</mark>",{}, 1000,+ true, false, true, "", 1000).get();++ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ(1, results["hits"][0]["highlights"].size());+ ASSERT_EQ("name", results["hits"][0]["highlights"][0]["field"]);+ ASSERT_EQ("Yes, <mark>C++</mark> is great!", results["hits"][0]["highlights"][0]["snippet"]);++ // without custom symbols, + should not be indexed, so the "C" record will show up first++ Collection* coll2 = collectionManager.create_collection("coll2", 1, fields, "points", 0, "").get();++ ASSERT_TRUE(coll2->add(doc1.dump()).ok());+ ASSERT_TRUE(coll2->add(doc2.dump()).ok());++ results = coll2->search("c++", {"name"},+ "", {}, {}, {0}, 10,+ 1, FREQUENCY, {false},+ 1, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1, {}, {}, {}, 0,+ "<mark>", "</mark>",{}, 1000,+ true, false, true, "", 1000).get();++ ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());++ // ensure that symbols are validated++ nlohmann::json coll_def;+ coll_def["fields"] = {+ {{"name", "foo"}, {"type", "string"}, {"facet", false}}+ };+ coll_def["name"] = "foo";+ coll_def["symbols_to_index"] = {"foo"};++ auto coll_op = collectionManager.create_collection(coll_def);++ ASSERT_FALSE(coll_op.ok());+ ASSERT_EQ("`symbols_to_index` should be an array of character symbols.", coll_op.error());++ coll_def["symbols_to_index"] = "f";+ coll_op = collectionManager.create_collection(coll_def);++ ASSERT_FALSE(coll_op.ok());+ ASSERT_EQ("`symbols_to_index` should be an array of character symbols.", coll_op.error());++ coll_def["symbols_to_index"] = 123;+ coll_op = collectionManager.create_collection(coll_def);++ ASSERT_FALSE(coll_op.ok());+ ASSERT_EQ("`symbols_to_index` should be an array of character symbols.", coll_op.error());++ collectionManager.drop_collection("coll1");+}diff --git a/test/index_test.cpp b/test/index_test.cppindex 997d50e1..957b376c 100644--- a/test/index_test.cpp+++ b/test/index_test.cpp@@ -10,7 +10,7 @@ TEST(IndexTest, ScrubReindexDoc) {search_schema.emplace("cast", field("cast", field_types::STRING_ARRAY, false));search_schema.emplace("movie", field("movie", field_types::BOOL, false));- Index index("index", search_schema, {}, {});+ Index index("index", search_schema, {}, {}, {}, {});nlohmann::json old_doc;old_doc["id"] = "1";old_doc["title"] = "One more thing.";diff --git a/test/tokenizer_test.cpp b/test/tokenizer_test.cppindex b1af9d99..1fa43f78 100644--- a/test/tokenizer_test.cpp+++ b/test/tokenizer_test.cpp@@ -125,7 +125,7 @@ TEST(TokenizerTest, ShouldTokenizeIteratively) {ASSERT_STREQ("everybody", tokens[3].c_str());ASSERT_STREQ("welcome", tokens[4].c_str());- // check for index when separators are not kept+ // check for index when token_separators are not keptTokenizer tokenizer2(withnewline, true, false);size_t expected_token_index = 0;std::vector<std::string> expected_tokens = {"michael", "jordan", "welcome", "everybody", "welcome"};@@ -192,7 +192,7 @@ TEST(TokenizerTest, ShouldTokenizeChineseText) {ASSERT_EQ("时间", tokens[4]);ASSERT_EQ("而", tokens[5]);- // with separators+ // with token_separatorstokens.clear();Tokenizer("很久以前,傳說在臺中北屯的一個地方", false, false, "zh").tokenize(tokens);ASSERT_EQ(10, tokens.size());