Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Make symbols for indexing and segmentation configurable.

Kishore Nallan07d838e38543
diff --git a/include/collection.h b/include/collection.h
index 4d3d5c74..065038bf 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -325,6 +325,10 @@ private:
std::vector<field> dynamic_fields;
+ std::vector<char> symbols_to_index;
+
+ std::vector<char> token_separators;
+
const std::vector<Index*> indices;
// methods
@@ -389,6 +393,8 @@ private:
std::vector<Index *> init_indices();
+ static std::vector<char> to_char_array(std::vector<std::string> strs);
+
public:
enum {MAX_ARRAY_MATCHES = 5};
@@ -413,10 +419,8 @@ public:
static constexpr const char* COLLECTION_NUM_MEMORY_SHARDS = "num_memory_shards";
static constexpr const char* COLLECTION_FALLBACK_FIELD_TYPE = "fallback_field_type";
- // DON'T CHANGE THESE VALUES!
- // this key is used as namespace key to store metadata about the document
- static constexpr const char* DOC_META_KEY = "$TSM$_";
- static constexpr const char* DOC_META_DIRTY_VALUES_KEY = "dirty_values";
+ static constexpr const char* COLLECTION_SYMBOLS_TO_INDEX = "symbols_to_index";
+ static constexpr const char* COLLECTION_SEPARATORS = "token_separators";
// methods
@@ -425,7 +429,8 @@ public:
Collection(const std::string& name, const uint32_t collection_id, const uint64_t created_at,
const uint32_t next_seq_id, Store *store, const std::vector<field>& fields,
const std::string& default_sorting_field, const size_t num_memory_shards,
- const float max_memory_ratio, const std::string& fallback_field_type);
+ const float max_memory_ratio, const std::string& fallback_field_type,
+ const std::vector<std::string>& symbols_to_index, const std::vector<std::string>& token_separators);
~Collection();
@@ -556,6 +561,10 @@ public:
DIRTY_VALUES parse_dirty_values_option(std::string& dirty_values) const;
+ std::vector<char> get_symbols_to_index();
+
+ std::vector<char> get_token_separators();
+
// Override operations
Option<uint32_t> add_override(const override_t & override);
diff --git a/include/collection_manager.h b/include/collection_manager.h
index 58261887..8027755c 100644
--- a/include/collection_manager.h
+++ b/include/collection_manager.h
@@ -134,7 +134,9 @@ public:
const std::vector<field> & fields,
const std::string & default_sorting_field="",
const uint64_t created_at = static_cast<uint64_t>(std::time(nullptr)),
- const std::string& fallback_field_type = "");
+ const std::string& fallback_field_type = "",
+ const std::vector<std::string>& symbols_to_index = {},
+ const std::vector<std::string>& token_separators = {});
locked_resource_view_t<Collection> get_collection(const std::string & collection_name) const;
diff --git a/include/index.h b/include/index.h
index 39dfe6aa..77f447cd 100644
--- a/include/index.h
+++ b/include/index.h
@@ -189,6 +189,10 @@ private:
// this is used for wildcard queries
sorted_array seq_ids;
+ std::vector<char> symbols_to_index;
+
+ std::vector<char> token_separators;
+
StringUtils string_utils;
// used as sentinels
@@ -312,7 +316,8 @@ public:
Index() = delete;
Index(const std::string name, const std::unordered_map<std::string, field> & search_schema,
- std::map<std::string, field> facet_schema, std::unordered_map<std::string, field> sort_schema);
+ std::map<std::string, field> facet_schema, std::unordered_map<std::string, field> sort_schema,
+ const std::vector<char>& symbols_to_index, const std::vector<char>& token_separators);
~Index();
diff --git a/include/tokenizer.h b/include/tokenizer.h
index 04b7ef4e..ba9e101f 100644
--- a/include/tokenizer.h
+++ b/include/tokenizer.h
@@ -24,6 +24,7 @@ private:
static const size_t SKIP = 2;
uint8_t index_symbols[256] = {};
+ uint8_t separator_symbols[256] = {};
std::string out;
@@ -42,7 +43,7 @@ private:
inline size_t get_stream_mode(char c) {
return (std::isalnum(c) || index_symbols[uint8_t(c)] == 1) ? INDEX : (
- (c == ' ' || c == '\n') ? SEPARATE : SKIP
+ (c == ' ' || c == '\n' || separator_symbols[uint8_t(c)] == 1) ? SEPARATE : SKIP
);
}
@@ -55,7 +56,8 @@ public:
explicit Tokenizer(const std::string& input,
bool normalize=true, bool no_op=false,
const std::string& locale = "",
- const std::vector<char>& symbols_to_index = {});
+ const std::vector<char>& symbols_to_index = {},
+ const std::vector<char>& separators = {});
~Tokenizer() {
iconv_close(cd);
diff --git a/src/collection.cpp b/src/collection.cpp
index 0f5564e6..b1017d16 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -44,13 +44,15 @@ struct match_index_t {
Collection::Collection(const std::string& name, const uint32_t collection_id, const uint64_t created_at,
const uint32_t next_seq_id, Store *store, const std::vector<field> &fields,
const std::string& default_sorting_field, const size_t num_memory_shards,
- const float max_memory_ratio, const std::string& fallback_field_type):
+ const float max_memory_ratio, const std::string& fallback_field_type,
+ const std::vector<std::string>& symbols_to_index, const std::vector<std::string>& token_separators):
name(name), collection_id(collection_id), created_at(created_at),
next_seq_id(next_seq_id), store(store),
fields(fields), default_sorting_field(default_sorting_field),
num_memory_shards(num_memory_shards),
max_memory_ratio(max_memory_ratio),
fallback_field_type(fallback_field_type), dynamic_fields({}),
+ symbols_to_index(to_char_array(symbols_to_index)), token_separators(to_char_array(token_separators)),
indices(init_indices()) {
this->num_documents = 0;
@@ -1369,7 +1371,10 @@ void Collection::parse_search_query(const std::string &query, std::vector<std::s
if(already_segmented) {
StringUtils::split(query, tokens, " ");
} else {
- Tokenizer(query, true, false, locale, {'-'}).tokenize(tokens);
+ std::vector<char> custom_symbols = symbols_to_index;
+ custom_symbols.push_back('-');
+
+ Tokenizer(query, true, false, locale, custom_symbols).tokenize(tokens);
}
bool exclude_operator_prior = false;
@@ -1378,17 +1383,24 @@ void Collection::parse_search_query(const std::string &query, std::vector<std::s
if(token[0] == '-') {
exclude_operator_prior = true;
token = token.substr(1);
- } else {
- // remove hyphens (added earlier for supporting negation syntax)
- // to maintain parity with indexing which currently removing special symbols
- token.erase(std::remove(token.begin(), token.end(), '-'), token.end());
}
- if(exclude_operator_prior) {
- q_exclude_tokens.push_back(token);
- exclude_operator_prior = false;
+ // retokenize using collection config
+ std::vector<std::string> sub_tokens;
+
+ if(already_segmented) {
+ StringUtils::split(token, sub_tokens, " ");
} else {
- q_include_tokens.push_back(token);
+ Tokenizer(token, true, false, locale, symbols_to_index, token_separators).tokenize(sub_tokens);
+ }
+
+ for(auto& sub_token: sub_tokens) {
+ if(exclude_operator_prior) {
+ q_exclude_tokens.push_back(sub_token);
+ exclude_operator_prior = false;
+ } else {
+ q_include_tokens.push_back(sub_token);
+ }
}
}
@@ -1673,7 +1685,7 @@ void Collection::highlight_result(const field &search_field,
text = document[search_field.name][match_index.index];
}
- Tokenizer tokenizer(text, true, false, search_field.locale);
+ Tokenizer tokenizer(text, true, false, search_field.locale, symbols_to_index, token_separators);
if(search_field.locale == "ko") {
text = string_utils.unicode_nfkd(text);
@@ -2777,7 +2789,8 @@ std::vector<Index *> Collection::init_indices() {
}
for(size_t i = 0; i < num_memory_shards; i++) {
- Index* index = new Index(name+std::to_string(i), search_schema, facet_schema, sort_schema);
+ Index* index = new Index(name+std::to_string(i), search_schema, facet_schema, sort_schema,
+ symbols_to_index, token_separators);
index_list.push_back(index);
}
@@ -2800,3 +2813,22 @@ DIRTY_VALUES Collection::parse_dirty_values_option(std::string& dirty_values) co
return dirty_values_action;
}
+
+std::vector<char> Collection::to_char_array(std::vector<std::string> strings) {
+ std::vector<char> vec;
+ for(const auto& s: strings) {
+ if(s.length() == 1) {
+ vec.push_back(s[0]);
+ }
+ }
+
+ return vec;
+}
+
+std::vector<char> Collection::get_symbols_to_index() {
+ return symbols_to_index;
+}
+
+std::vector<char> Collection::get_token_separators() {
+ return token_separators;
+}
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index de6ed66a..b11edbf3 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -52,6 +52,17 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collection
collection_meta[Collection::COLLECTION_FALLBACK_FIELD_TYPE].get<std::string>() :
"";
+ std::vector<std::string> symbols_to_index;
+ std::vector<std::string> token_separators;
+
+ if(collection_meta.count(Collection::COLLECTION_SYMBOLS_TO_INDEX) != 0) {
+ symbols_to_index = collection_meta[Collection::COLLECTION_SYMBOLS_TO_INDEX].get<std::vector<std::string>>();
+ }
+
+ if(collection_meta.count(Collection::COLLECTION_SEPARATORS) != 0) {
+ token_separators = collection_meta[Collection::COLLECTION_SEPARATORS].get<std::vector<std::string>>();
+ }
+
LOG(INFO) << "Found collection " << this_collection_name << " with " << num_memory_shards << " memory shards.";
Collection* collection = new Collection(this_collection_name,
@@ -63,7 +74,9 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collection
default_sorting_field,
num_memory_shards,
max_memory_ratio,
- fallback_field_type);
+ fallback_field_type,
+ symbols_to_index,
+ token_separators);
return collection;
}
@@ -222,7 +235,9 @@ Option<Collection*> CollectionManager::create_collection(const std::string& name
const std::vector<field> & fields,
const std::string& default_sorting_field,
const uint64_t created_at,
- const std::string& fallback_field_type) {
+ const std::string& fallback_field_type,
+ const std::vector<std::string>& symbols_to_index,
+ const std::vector<std::string>& token_separators) {
if(store->contains(Collection::get_meta_key(name))) {
return Option<Collection*>(409, std::string("A collection with name `") + name + "` already exists.");
@@ -252,10 +267,13 @@ Option<Collection*> CollectionManager::create_collection(const std::string& name
collection_meta[Collection::COLLECTION_CREATED] = created_at;
collection_meta[Collection::COLLECTION_NUM_MEMORY_SHARDS] = num_memory_shards;
collection_meta[Collection::COLLECTION_FALLBACK_FIELD_TYPE] = fallback_field_type;
+ collection_meta[Collection::COLLECTION_SYMBOLS_TO_INDEX] = symbols_to_index;
+ collection_meta[Collection::COLLECTION_SEPARATORS] = token_separators;
Collection* new_collection = new Collection(name, next_collection_id, created_at, 0, store, fields,
default_sorting_field, num_memory_shards,
- this->max_memory_ratio, fallback_field_type);
+ this->max_memory_ratio, fallback_field_type,
+ symbols_to_index, token_separators);
next_collection_id++;
rocksdb::WriteBatch batch;
@@ -838,6 +856,8 @@ nlohmann::json CollectionManager::get_collection_summaries() const {
Option<Collection*> CollectionManager::create_collection(nlohmann::json& req_json) {
const char* NUM_MEMORY_SHARDS = "num_memory_shards";
+ const char* SYMBOLS_TO_INDEX = "symbols_to_index";
+ const char* TOKEN_SEPARATORS = "token_separators";
const char* DEFAULT_SORTING_FIELD = "default_sorting_field";
// validate presence of mandatory fields
@@ -854,6 +874,14 @@ Option<Collection*> CollectionManager::create_collection(nlohmann::json& req_jso
req_json[NUM_MEMORY_SHARDS] = CollectionManager::DEFAULT_NUM_MEMORY_SHARDS;
}
+ if(req_json.count(SYMBOLS_TO_INDEX) == 0) {
+ req_json[SYMBOLS_TO_INDEX] = std::vector<std::string>();
+ }
+
+ if(req_json.count(TOKEN_SEPARATORS) == 0) {
+ req_json[TOKEN_SEPARATORS] = std::vector<std::string>();
+ }
+
if(req_json.count("fields") == 0) {
return Option<Collection*>(400, "Parameter `fields` is required.");
}
@@ -871,6 +899,26 @@ Option<Collection*> CollectionManager::create_collection(nlohmann::json& req_jso
return Option<Collection*>(400, std::string("`") + NUM_MEMORY_SHARDS + "` should be a positive integer.");
}
+ if(!req_json[SYMBOLS_TO_INDEX].is_array()) {
+ return Option<Collection*>(400, std::string("`") + SYMBOLS_TO_INDEX + "` should be an array of character symbols.");
+ }
+
+ if(!req_json[TOKEN_SEPARATORS].is_array()) {
+ return Option<Collection*>(400, std::string("`") + TOKEN_SEPARATORS + "` should be an array of character symbols.");
+ }
+
+ for (auto it = req_json[SYMBOLS_TO_INDEX].begin(); it != req_json[SYMBOLS_TO_INDEX].end(); ++it) {
+ if(!it->is_string() || it->get<std::string>().size() != 1 ) {
+ return Option<Collection*>(400, std::string("`") + SYMBOLS_TO_INDEX + "` should be an array of character symbols.");
+ }
+ }
+
+ for (auto it = req_json[TOKEN_SEPARATORS].begin(); it != req_json[TOKEN_SEPARATORS].end(); ++it) {
+ if(!it->is_string() || it->get<std::string>().size() != 1 ) {
+ return Option<Collection*>(400, std::string("`") + TOKEN_SEPARATORS + "` should be an array of character symbols.");
+ }
+ }
+
size_t num_memory_shards = req_json[NUM_MEMORY_SHARDS].get<size_t>();
if(num_memory_shards == 0) {
return Option<Collection*>(400, std::string("`") + NUM_MEMORY_SHARDS + "` should be a positive integer.");
@@ -897,7 +945,9 @@ Option<Collection*> CollectionManager::create_collection(nlohmann::json& req_jso
return CollectionManager::get_instance().create_collection(req_json["name"], num_memory_shards,
fields, default_sorting_field, created_at,
- fallback_field_type);
+ fallback_field_type,
+ req_json[SYMBOLS_TO_INDEX],
+ req_json[TOKEN_SEPARATORS]);
}
Option<bool> CollectionManager::load_collection(const nlohmann::json &collection_meta,
diff --git a/src/index.cpp b/src/index.cpp
index 9fb2c182..467ff762 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -24,8 +24,10 @@ spp::sparse_hash_map<uint32_t, int64_t> Index::text_match_sentinel_value;
spp::sparse_hash_map<uint32_t, int64_t> Index::seq_id_sentinel_value;
Index::Index(const std::string name, const std::unordered_map<std::string, field> & search_schema,
- std::map<std::string, field> facet_schema, std::unordered_map<std::string, field> sort_schema):
- name(name), search_schema(search_schema), facet_schema(facet_schema), sort_schema(sort_schema) {
+ std::map<std::string, field> facet_schema, std::unordered_map<std::string, field> sort_schema,
+ const std::vector<char>& symbols_to_index, const std::vector<char>& token_separators):
+ name(name), search_schema(search_schema), facet_schema(facet_schema), sort_schema(sort_schema),
+ symbols_to_index(symbols_to_index), token_separators(token_separators) {
for(const auto & fname_field: search_schema) {
if(fname_field.second.is_string()) {
@@ -542,7 +544,7 @@ void Index::index_string_field(const std::string & text, const int64_t score, ar
uint32_t seq_id, bool is_facet, const field & a_field) {
std::unordered_map<std::string, std::vector<uint32_t>> token_to_offsets;
- Tokenizer tokenizer(text, true, !a_field.is_string(), a_field.locale);
+ Tokenizer tokenizer(text, true, !a_field.is_string(), a_field.locale, symbols_to_index, token_separators);
std::string token, last_token;
size_t token_index = 0;
@@ -597,7 +599,7 @@ void Index::index_string_array_field(const std::vector<std::string> & strings, c
const std::string& str = strings[array_index];
std::set<std::string> token_set; // required to deal with repeating tokens
- Tokenizer tokenizer(str, true, !a_field.is_string(), a_field.locale);
+ Tokenizer tokenizer(str, true, !a_field.is_string(), a_field.locale, symbols_to_index, token_separators);
std::string token, last_token;
size_t token_index = 0;
diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
index 3ce7b7bd..1d868e01 100644
--- a/src/tokenizer.cpp
+++ b/src/tokenizer.cpp
@@ -3,7 +3,8 @@
#include "tokenizer.h"
Tokenizer::Tokenizer(const std::string& input, bool normalize, bool no_op, const std::string& locale,
- const std::vector<char>& symbols_to_index):
+ const std::vector<char>& symbols_to_index,
+ const std::vector<char>& separators):
i(0), normalize(normalize), no_op(no_op), locale(locale) {
if(locale == "zh") {
@@ -50,6 +51,10 @@ Tokenizer::Tokenizer(const std::string& input, bool normalize, bool no_op, const
index_symbols[uint8_t(c)] = 1;
}
+ for(char c: separators) {
+ separator_symbols[uint8_t(c)] = 1;
+ }
+
UErrorCode errcode = U_ZERO_ERROR;
nfkd = icu::Normalizer2::getNFKDInstance(errcode);
}
diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp
index ac87bfc4..edc11c58 100644
--- a/test/collection_manager_test.cpp
+++ b/test/collection_manager_test.cpp
@@ -33,7 +33,8 @@ protected:
};
sort_fields = { sort_by("points", "DESC") };
- collection1 = collectionManager.create_collection("collection1", 4, search_fields, "points", 12345).get();
+ collection1 = collectionManager.create_collection("collection1", 4, search_fields,
+ "points", 12345, "", {"+"}, {"-"}).get();
}
virtual void SetUp() {
@@ -94,7 +95,7 @@ TEST_F(CollectionManagerTest, CollectionCreation) {
"{\"facet\":true,\"locale\":\"\",\"name\":\".*_year\",\"optional\":true,\"type\":\"int32\"},"
"{\"facet\":false,\"locale\":\"\",\"name\":\"location\",\"optional\":true,\"type\":\"geopoint\"},"
"{\"facet\":false,\"locale\":\"\",\"name\":\"points\",\"optional\":false,\"type\":\"int32\"}],\"id\":0,"
- "\"name\":\"collection1\",\"num_memory_shards\":4}",
+ "\"name\":\"collection1\",\"num_memory_shards\":4,\"symbols_to_index\":[\"+\"],\"token_separators\":[\"-\"]}",
collection_meta_json);
ASSERT_EQ("1", next_collection_id);
}
@@ -113,16 +114,27 @@ TEST_F(CollectionManagerTest, ShouldInitCollection) {
delete collection;
- // with created_at ts
+ // with non-default values
nlohmann::json collection_meta2 =
nlohmann::json::parse("{\"name\": \"foobar\", \"id\": 100, \"fields\": [{\"name\": \"org\", \"type\": "
"\"string\", \"facet\": false}], \"created_at\": 12345,"
- "\"default_sorting_field\": \"foo\"}");
+ "\"default_sorting_field\": \"foo\","
+ "\"symbols_to_index\": [\"+\"], \"token_separators\": [\"-\"]}");
+
collection = collectionManager.init_collection(collection_meta2, 100, store, 1.0f);
ASSERT_EQ(12345, collection->get_created_at());
+ std::vector<char> expected_symbols = {'+'};
+ std::vector<char> expected_separators = {'-'};
+
+ ASSERT_EQ(1, collection->get_token_separators().size());
+ ASSERT_EQ('-', collection->get_token_separators()[0]);
+
+ ASSERT_EQ(1, collection->get_symbols_to_index().size());
+ ASSERT_EQ('+', collection->get_symbols_to_index()[0]);
+
delete collection;
}
@@ -282,6 +294,15 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {
ASSERT_EQ(0, synonyms.at("id3").root.size());
ASSERT_EQ(2, synonyms.at("id3").synonyms.size());
+ std::vector<char> expected_symbols = {'+'};
+ std::vector<char> expected_separators = {'-'};
+
+ ASSERT_EQ(1, collection1->get_token_separators().size());
+ ASSERT_EQ('-', collection1->get_token_separators()[0]);
+
+ ASSERT_EQ(1, collection1->get_symbols_to_index().size());
+ ASSERT_EQ('+', collection1->get_symbols_to_index()[0]);
+
results = collection1->search("thomas", search_fields, "", facets, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();
ASSERT_EQ(4, results["hits"].size());
}
diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cpp
index 55ab32a4..3f7e291a 100644
--- a/test/collection_specific_test.cpp
+++ b/test/collection_specific_test.cpp
@@ -952,4 +952,144 @@ TEST_F(CollectionSpecificTest, HighlightEmptyArray) {
ASSERT_EQ("name", results["hits"][0]["highlights"][0]["field"]);
collectionManager.drop_collection("coll1");
-}
\ No newline at end of file
+}
+
+TEST_F(CollectionSpecificTest, CustomSeparators) {
+ std::vector<field> fields = {field("name", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ Collection* coll1 = collectionManager.create_collection(
+ "coll1", 1, fields, "points", 0, "", {}, {"-"}
+ ).get();
+
+ nlohmann::json doc1;
+ doc1["id"] = "0";
+ doc1["name"] = "alpha-beta-gamma-omega-zeta";
+ doc1["points"] = 100;
+
+ ASSERT_TRUE(coll1->add(doc1.dump()).ok());
+
+ auto results = coll1->search("gamma", {"name"},
+ "", {}, {}, {0}, 10,
+ 1, FREQUENCY, {false},
+ 1, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1, {}, {}, {}, 0,
+ "<mark>", "</mark>",{}, 1000,
+ true, false, true, "", 1000).get();
+
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ(1, results["hits"][0]["highlights"].size());
+ ASSERT_EQ("name", results["hits"][0]["highlights"][0]["field"]);
+ ASSERT_EQ("alpha-beta-<mark>gamma</mark>-omega-zeta", results["hits"][0]["highlights"][0]["snippet"]);
+
+ // ensure that symbols are validated
+
+ nlohmann::json coll_def;
+ coll_def["fields"] = {
+ {{"name", "foo"}, {"type", "string"}, {"facet", false}}
+ };
+ coll_def["name"] = "foo";
+ coll_def["token_separators"] = {"foo"};
+
+ auto coll_op = collectionManager.create_collection(coll_def);
+
+ ASSERT_FALSE(coll_op.ok());
+ ASSERT_EQ("`token_separators` should be an array of character symbols.", coll_op.error());
+
+ coll_def["token_separators"] = "f";
+ coll_op = collectionManager.create_collection(coll_def);
+
+ ASSERT_FALSE(coll_op.ok());
+ ASSERT_EQ("`token_separators` should be an array of character symbols.", coll_op.error());
+
+ coll_def["token_separators"] = 123;
+ coll_op = collectionManager.create_collection(coll_def);
+
+ ASSERT_FALSE(coll_op.ok());
+ ASSERT_EQ("`token_separators` should be an array of character symbols.", coll_op.error());
+
+ collectionManager.drop_collection("coll1");
+}
+
+TEST_F(CollectionSpecificTest, CustomSymbolsForIndexing) {
+ std::vector<field> fields = {field("name", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ Collection* coll1 = collectionManager.create_collection(
+ "coll1", 1, fields, "points", 0, "", {"+"}, {}
+ ).get();
+
+ nlohmann::json doc1;
+ doc1["id"] = "0";
+ doc1["name"] = "Yes, C++ is great!";
+ doc1["points"] = 100;
+
+ nlohmann::json doc2;
+ doc2["id"] = "1";
+ doc2["name"] = "Yes, C is great!";
+ doc2["points"] = 100;
+
+ ASSERT_TRUE(coll1->add(doc1.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());
+
+ auto results = coll1->search("c++", {"name"},
+ "", {}, {}, {0}, 10,
+ 1, FREQUENCY, {false},
+ 1, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1, {}, {}, {}, 0,
+ "<mark>", "</mark>",{}, 1000,
+ true, false, true, "", 1000).get();
+
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ(1, results["hits"][0]["highlights"].size());
+ ASSERT_EQ("name", results["hits"][0]["highlights"][0]["field"]);
+ ASSERT_EQ("Yes, <mark>C++</mark> is great!", results["hits"][0]["highlights"][0]["snippet"]);
+
+ // without custom symbols, + should not be indexed, so the "C" record will show up first
+
+ Collection* coll2 = collectionManager.create_collection("coll2", 1, fields, "points", 0, "").get();
+
+ ASSERT_TRUE(coll2->add(doc1.dump()).ok());
+ ASSERT_TRUE(coll2->add(doc2.dump()).ok());
+
+ results = coll2->search("c++", {"name"},
+ "", {}, {}, {0}, 10,
+ 1, FREQUENCY, {false},
+ 1, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1, {}, {}, {}, 0,
+ "<mark>", "</mark>",{}, 1000,
+ true, false, true, "", 1000).get();
+
+ ASSERT_EQ(2, results["hits"].size());
+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
+
+ // ensure that symbols are validated
+
+ nlohmann::json coll_def;
+ coll_def["fields"] = {
+ {{"name", "foo"}, {"type", "string"}, {"facet", false}}
+ };
+ coll_def["name"] = "foo";
+ coll_def["symbols_to_index"] = {"foo"};
+
+ auto coll_op = collectionManager.create_collection(coll_def);
+
+ ASSERT_FALSE(coll_op.ok());
+ ASSERT_EQ("`symbols_to_index` should be an array of character symbols.", coll_op.error());
+
+ coll_def["symbols_to_index"] = "f";
+ coll_op = collectionManager.create_collection(coll_def);
+
+ ASSERT_FALSE(coll_op.ok());
+ ASSERT_EQ("`symbols_to_index` should be an array of character symbols.", coll_op.error());
+
+ coll_def["symbols_to_index"] = 123;
+ coll_op = collectionManager.create_collection(coll_def);
+
+ ASSERT_FALSE(coll_op.ok());
+ ASSERT_EQ("`symbols_to_index` should be an array of character symbols.", coll_op.error());
+
+ collectionManager.drop_collection("coll1");
+}
diff --git a/test/index_test.cpp b/test/index_test.cpp
index 997d50e1..957b376c 100644
--- a/test/index_test.cpp
+++ b/test/index_test.cpp
@@ -10,7 +10,7 @@ TEST(IndexTest, ScrubReindexDoc) {
search_schema.emplace("cast", field("cast", field_types::STRING_ARRAY, false));
search_schema.emplace("movie", field("movie", field_types::BOOL, false));
- Index index("index", search_schema, {}, {});
+ Index index("index", search_schema, {}, {}, {}, {});
nlohmann::json old_doc;
old_doc["id"] = "1";
old_doc["title"] = "One more thing.";
diff --git a/test/tokenizer_test.cpp b/test/tokenizer_test.cpp
index b1af9d99..1fa43f78 100644
--- a/test/tokenizer_test.cpp
+++ b/test/tokenizer_test.cpp
@@ -125,7 +125,7 @@ TEST(TokenizerTest, ShouldTokenizeIteratively) {
ASSERT_STREQ("everybody", tokens[3].c_str());
ASSERT_STREQ("welcome", tokens[4].c_str());
- // check for index when separators are not kept
+ // check for index when token_separators are not kept
Tokenizer tokenizer2(withnewline, true, false);
size_t expected_token_index = 0;
std::vector<std::string> expected_tokens = {"michael", "jordan", "welcome", "everybody", "welcome"};
@@ -192,7 +192,7 @@ TEST(TokenizerTest, ShouldTokenizeChineseText) {
ASSERT_EQ("时间", tokens[4]);
ASSERT_EQ("而", tokens[5]);
- // with separators
+ // with token_separators
tokens.clear();
Tokenizer("很久以前,傳說在臺中北屯的一個地方", false, false, "zh").tokenize(tokens);
ASSERT_EQ(10, tokens.size());