Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Remove facet schema book keeping datastructure.
Kishore Nallanc140d42f80d3
diff --git a/include/collection.h b/include/collection.hindex ad5e26f0..5555c1d9 100644--- a/include/collection.h+++ b/include/collection.h@@ -174,8 +174,6 @@ private:std::unordered_map<std::string, field> search_schema;- std::map<std::string, field> facet_schema; // std::map guarantees order of fields-std::unordered_map<std::string, field> sort_schema;std::map<std::string, override_t> overrides;diff --git a/include/index.h b/include/index.hindex d62b2e2e..28387969 100644--- a/include/index.h+++ b/include/index.h@@ -430,8 +430,6 @@ private:std::unordered_map<std::string, field> search_schema;- std::map<std::string, field> facet_schema; // std::map guarantees order of fields-std::unordered_map<std::string, field> sort_schema;spp::sparse_hash_map<std::string, art_tree*> search_index;@@ -635,7 +633,6 @@ public:const Store* store,ThreadPool* thread_pool,const std::unordered_map<std::string, field>& search_schema,- const std::map<std::string, field>& facet_schema,const std::unordered_map<std::string, field>& sort_schema,const std::vector<char>& symbols_to_index, const std::vector<char>& token_separators);@@ -679,7 +676,6 @@ public:static void compute_token_offsets_facets(index_record& record,const std::unordered_map<std::string, field>& search_schema,- const std::map<std::string, field>& facet_schema,const std::vector<char>& local_token_separators,const std::vector<char>& local_symbols_to_index);@@ -731,7 +727,6 @@ public:const size_t batch_start_index, const size_t batch_size,const std::string & default_sorting_field,const std::unordered_map<std::string, field> & search_schema,- const std::map<std::string, field> & facet_schema,const std::string& fallback_field_type,const std::vector<char>& token_separators,const std::vector<char>& symbols_to_index);@@ -740,7 +735,6 @@ public:std::vector<index_record> & iter_batch,const std::string & default_sorting_field,const std::unordered_map<std::string, field> & search_schema,- const std::map<std::string, field> & facet_schema,const std::string& fallback_field_type,const std::vector<char>& token_separators,const std::vector<char>& symbols_to_index);@@ -768,7 +762,6 @@ public:static Option<uint32_t> validate_index_in_memory(nlohmann::json &document, uint32_t seq_id,const std::string & default_sorting_field,const std::unordered_map<std::string, field> & search_schema,- const std::map<std::string, field> & facet_schema,const index_operation_t op,const std::string& fallback_field_type,const DIRTY_VALUES& dirty_values);diff --git a/src/collection.cpp b/src/collection.cppindex 8a82a7c8..cc6349e9 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -354,7 +354,7 @@ Option<uint32_t> Collection::index_in_memory(nlohmann::json &document, uint32_tstd::unique_lock lock(mutex);Option<uint32_t> validation_op = Index::validate_index_in_memory(document, seq_id, default_sorting_field,- search_schema, facet_schema, op,+ search_schema, op,fallback_field_type, dirty_values);if(!validation_op.ok()) {@@ -362,11 +362,11 @@ Option<uint32_t> Collection::index_in_memory(nlohmann::json &document, uint32_t}index_record rec(0, seq_id, document, op, dirty_values);- Index::compute_token_offsets_facets(rec, search_schema, facet_schema, token_separators, symbols_to_index);+ Index::compute_token_offsets_facets(rec, search_schema, token_separators, symbols_to_index);std::vector<index_record> index_batch;index_batch.emplace_back(std::move(rec));- Index::batch_memory_index(index, index_batch, default_sorting_field, search_schema, facet_schema,+ Index::batch_memory_index(index, index_batch, default_sorting_field, search_schema,fallback_field_type, token_separators, symbols_to_index);num_documents += 1;@@ -376,7 +376,7 @@ Option<uint32_t> Collection::index_in_memory(nlohmann::json &document, uint32_tsize_t Collection::batch_index_in_memory(std::vector<index_record>& index_records) {std::unique_lock lock(mutex);size_t num_indexed = Index::batch_memory_index(index, index_records, default_sorting_field,- search_schema, facet_schema, fallback_field_type,+ search_schema, fallback_field_type,token_separators, symbols_to_index);num_documents += num_indexed;return num_indexed;@@ -804,7 +804,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s// validate facet fieldsfor(const std::string & field_name: facet_fields) {- if(facet_schema.count(field_name) == 0) {+ if(search_schema.count(field_name) == 0 || !search_schema.at(field_name).facet) {std::string error = "Could not find a facet field named `" + field_name + "` in the schema.";return Option<nlohmann::json>(404, error);}@@ -845,7 +845,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const sreturn Option<nlohmann::json>(400, error);}- if(facet_schema.count(facet_query.field_name) == 0) {+ if(search_schema.count(facet_query.field_name) == 0 || !search_schema.at(facet_query.field_name).facet) {std::string error = "Could not find a facet field named `" + facet_query.field_name + "` in the schema.";return Option<nlohmann::json>(404, error);}@@ -1664,52 +1664,52 @@ bool Collection::facet_value_to_string(const facet &a_facet, const facet_count_t}LOG(ERROR) << "Could not find field " << a_facet.field_name << " in document during faceting.";- LOG(ERROR) << "Facet field type: " << facet_schema.at(a_facet.field_name).type;+ LOG(ERROR) << "Facet field type: " << search_schema.at(a_facet.field_name).type;LOG(ERROR) << "Actual document: " << document;return false;}- if(facet_schema.at(a_facet.field_name).is_array()) {+ if(search_schema.at(a_facet.field_name).is_array()) {size_t array_sz = document[a_facet.field_name].size();if(facet_count.array_pos >= array_sz) {LOG(ERROR) << "Facet field array size " << array_sz << " lesser than array pos " << facet_count.array_pos<< " for facet field " << a_facet.field_name;- LOG(ERROR) << "Facet field type: " << facet_schema.at(a_facet.field_name).type;+ LOG(ERROR) << "Facet field type: " << search_schema.at(a_facet.field_name).type;LOG(ERROR) << "Actual document: " << document;return false;}}- if(facet_schema.at(a_facet.field_name).type == field_types::STRING) {+ if(search_schema.at(a_facet.field_name).type == field_types::STRING) {value = document[a_facet.field_name];- } else if(facet_schema.at(a_facet.field_name).type == field_types::STRING_ARRAY) {+ } else if(search_schema.at(a_facet.field_name).type == field_types::STRING_ARRAY) {value = document[a_facet.field_name][facet_count.array_pos];- } else if(facet_schema.at(a_facet.field_name).type == field_types::INT32) {+ } else if(search_schema.at(a_facet.field_name).type == field_types::INT32) {int32_t raw_val = document[a_facet.field_name].get<int32_t>();value = std::to_string(raw_val);- } else if(facet_schema.at(a_facet.field_name).type == field_types::INT32_ARRAY) {+ } else if(search_schema.at(a_facet.field_name).type == field_types::INT32_ARRAY) {int32_t raw_val = document[a_facet.field_name][facet_count.array_pos].get<int32_t>();value = std::to_string(raw_val);- } else if(facet_schema.at(a_facet.field_name).type == field_types::INT64) {+ } else if(search_schema.at(a_facet.field_name).type == field_types::INT64) {int64_t raw_val = document[a_facet.field_name].get<int64_t>();value = std::to_string(raw_val);- } else if(facet_schema.at(a_facet.field_name).type == field_types::INT64_ARRAY) {+ } else if(search_schema.at(a_facet.field_name).type == field_types::INT64_ARRAY) {int64_t raw_val = document[a_facet.field_name][facet_count.array_pos].get<int64_t>();value = std::to_string(raw_val);- } else if(facet_schema.at(a_facet.field_name).type == field_types::FLOAT) {+ } else if(search_schema.at(a_facet.field_name).type == field_types::FLOAT) {float raw_val = document[a_facet.field_name].get<float>();value = StringUtils::float_to_str(raw_val);if(value != "0") {value.erase ( value.find_last_not_of('0') + 1, std::string::npos ); // remove trailing zeros}- } else if(facet_schema.at(a_facet.field_name).type == field_types::FLOAT_ARRAY) {+ } else if(search_schema.at(a_facet.field_name).type == field_types::FLOAT_ARRAY) {float raw_val = document[a_facet.field_name][facet_count.array_pos].get<float>();value = StringUtils::float_to_str(raw_val);value.erase ( value.find_last_not_of('0') + 1, std::string::npos ); // remove trailing zeros- } else if(facet_schema.at(a_facet.field_name).type == field_types::BOOL) {+ } else if(search_schema.at(a_facet.field_name).type == field_types::BOOL) {value = std::to_string(document[a_facet.field_name].get<bool>());value = (value == "1") ? "true" : "false";- } else if(facet_schema.at(a_facet.field_name).type == field_types::BOOL_ARRAY) {+ } else if(search_schema.at(a_facet.field_name).type == field_types::BOOL_ARRAY) {value = std::to_string(document[a_facet.field_name][facet_count.array_pos].get<bool>());value = (value == "1") ? "true" : "false";}@@ -2282,8 +2282,10 @@ std::vector<std::string> Collection::get_facet_fields() {std::shared_lock lock(mutex);std::vector<std::string> facet_fields_copy;- for(auto it = facet_schema.begin(); it != facet_schema.end(); ++it) {- facet_fields_copy.push_back(it->first);+ for(auto it = search_schema.begin(); it != search_schema.end(); ++it) {+ if(it->second.facet) {+ facet_fields_copy.push_back(it->first);+ }}return facet_fields_copy;@@ -2701,10 +2703,6 @@ Option<bool> Collection::check_and_update_schema(nlohmann::json& document, const}search_schema.emplace(new_field.name, new_field);- if(new_field.is_facet()) {- facet_schema.emplace(new_field.name, new_field);- }-fields.emplace_back(new_field);}@@ -2760,10 +2758,6 @@ Index* Collection::init_index() {search_schema.emplace(field.name, field);- if(field.is_facet()) {- facet_schema.emplace(field.name, field);- }-if(field.is_sortable()) {sort_schema.emplace(field.name, field);}@@ -2773,7 +2767,7 @@ Index* Collection::init_index() {collection_id,store,CollectionManager::get_instance().get_thread_pool(),- search_schema, facet_schema, sort_schema,+ search_schema, sort_schema,symbols_to_index, token_separators);}diff --git a/src/index.cpp b/src/index.cppindex 154706e7..0a1b2c9a 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -40,10 +40,10 @@ spp::sparse_hash_map<uint32_t, int64_t> Index::str_sentinel_value;Index::Index(const std::string& name, const uint32_t collection_id, const Store* store, ThreadPool* thread_pool,const std::unordered_map<std::string, field> & search_schema,- const std::map<std::string, field>& facet_schema, const std::unordered_map<std::string, field>& sort_schema,+ const std::unordered_map<std::string, field>& sort_schema,const std::vector<char>& symbols_to_index, const std::vector<char>& token_separators):name(name), collection_id(collection_id), store(store), thread_pool(thread_pool),- search_schema(search_schema), facet_schema(facet_schema), sort_schema(sort_schema),+ search_schema(search_schema), sort_schema(sort_schema),seq_ids(new id_list_t(256)), symbols_to_index(symbols_to_index), token_separators(token_separators) {for(const auto & fname_field: search_schema) {@@ -66,6 +66,15 @@ Index::Index(const std::string& name, const uint32_t collection_id, const Store*numerical_index.emplace(fname_field.first, num_tree);}+ if(fname_field.second.facet) {+ array_mapped_facet_t facet_array;+ for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {+ facet_array[i] = new facet_map_t();+ }++ facet_index_v3.emplace(fname_field.first, facet_array);+ }+// initialize for non-string facet fieldsif(fname_field.second.facet && !fname_field.second.is_string()) {art_tree *ft = new art_tree;@@ -94,15 +103,6 @@ Index::Index(const std::string& name, const uint32_t collection_id, const Store*}}- for(const auto& pair: facet_schema) {- array_mapped_facet_t facet_array;- for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {- facet_array[i] = new facet_map_t();- }-- facet_index_v3.emplace(pair.first, facet_array);- }-num_documents = 0;}@@ -208,7 +208,6 @@ int64_t Index::float_to_in64_t(float f) {void Index::compute_token_offsets_facets(index_record& record,const std::unordered_map<std::string, field>& search_schema,- const std::map<std::string, field>& facet_schema,const std::vector<char>& local_token_separators,const std::vector<char>& local_symbols_to_index) {@@ -222,7 +221,7 @@ void Index::compute_token_offsets_facets(index_record& record,offsets_facet_hashes_t offset_facet_hashes;- bool is_facet = (facet_schema.count(field_name) != 0);+ bool is_facet = search_schema.at(field_name).facet;// non-string, non-geo faceted field should be indexed as faceted string field as wellif(field_pair.second.facet && !field_pair.second.is_string() && !field_pair.second.is_geopoint()) {@@ -291,7 +290,6 @@ void Index::compute_token_offsets_facets(index_record& record,Option<uint32_t> Index::validate_index_in_memory(nlohmann::json& document, uint32_t seq_id,const std::string & default_sorting_field,const std::unordered_map<std::string, field> & search_schema,- const std::map<std::string, field> & facet_schema,const index_operation_t op,const std::string& fallback_field_type,const DIRTY_VALUES& dirty_values) {@@ -443,7 +441,6 @@ void Index::validate_and_preprocess(Index *index, std::vector<index_record>& iteconst size_t batch_start_index, const size_t batch_size,const std::string& default_sorting_field,const std::unordered_map<std::string, field>& search_schema,- const std::map<std::string, field>& facet_schema,const std::string& fallback_field_type,const std::vector<char>& token_separators,const std::vector<char>& symbols_to_index) {@@ -465,7 +462,7 @@ void Index::validate_and_preprocess(Index *index, std::vector<index_record>& iteOption<uint32_t> validation_op = validate_index_in_memory(index_rec.doc, index_rec.seq_id,default_sorting_field,- search_schema, facet_schema,+ search_schema,index_rec.operation,fallback_field_type,index_rec.dirty_values);@@ -482,7 +479,7 @@ void Index::validate_and_preprocess(Index *index, std::vector<index_record>& itescrub_reindex_doc(search_schema, index_rec.doc, index_rec.del_doc, index_rec.old_doc);}- compute_token_offsets_facets(index_rec, search_schema, facet_schema, token_separators, symbols_to_index);+ compute_token_offsets_facets(index_rec, search_schema, token_separators, symbols_to_index);int64_t points = 0;@@ -514,7 +511,6 @@ void Index::validate_and_preprocess(Index *index, std::vector<index_record>& itesize_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_batch,const std::string & default_sorting_field,const std::unordered_map<std::string, field> & search_schema,- const std::map<std::string, field> & facet_schema,const std::string& fallback_field_type,const std::vector<char>& token_separators,const std::vector<char>& symbols_to_index) {@@ -543,8 +539,7 @@ size_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_bindex->thread_pool->enqueue([&, batch_index, batch_len]() {validate_and_preprocess(index, iter_batch, batch_index, batch_len, default_sorting_field, search_schema,- facet_schema, fallback_field_type,- token_separators, symbols_to_index);+ fallback_field_type, token_separators, symbols_to_index);std::unique_lock<std::mutex> lock(m_process);num_processed++;@@ -2921,7 +2916,7 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&facet_infos[findex].use_facet_query = false;- const field &facet_field = facet_schema.at(a_facet.field_name);+ const field &facet_field = search_schema.at(a_facet.field_name);facet_infos[findex].facet_field = facet_field;facet_infos[findex].should_compute_stats = (facet_field.type != field_types::STRING &&@@ -3944,8 +3939,6 @@ void Index::refresh_schemas(const std::vector<field>& new_fields) {}if(new_field.is_facet()) {- facet_schema.emplace(new_field.name, new_field);-array_mapped_facet_t facet_array;for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {facet_array[i] = new facet_map_t();diff --git a/test/index_test.cpp b/test/index_test.cppindex 554cd31b..5bff3b81 100644--- a/test/index_test.cpp+++ b/test/index_test.cpp@@ -12,7 +12,7 @@ TEST(IndexTest, ScrubReindexDoc) {ThreadPool pool(4);- Index index("index", 1, nullptr, &pool, search_schema, {}, {}, {}, {});+ Index index("index", 1, nullptr, &pool, search_schema, {}, {}, {});nlohmann::json old_doc;old_doc["id"] = "1";old_doc["title"] = "One more thing.";