Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Remove facet schema book keeping datastructure.

Kishore Nallanc140d42f80d3
diff --git a/include/collection.h b/include/collection.h
index ad5e26f0..5555c1d9 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -174,8 +174,6 @@ private:
std::unordered_map<std::string, field> search_schema;
- std::map<std::string, field> facet_schema; // std::map guarantees order of fields
-
std::unordered_map<std::string, field> sort_schema;
std::map<std::string, override_t> overrides;
diff --git a/include/index.h b/include/index.h
index d62b2e2e..28387969 100644
--- a/include/index.h
+++ b/include/index.h
@@ -430,8 +430,6 @@ private:
std::unordered_map<std::string, field> search_schema;
- std::map<std::string, field> facet_schema; // std::map guarantees order of fields
-
std::unordered_map<std::string, field> sort_schema;
spp::sparse_hash_map<std::string, art_tree*> search_index;
@@ -635,7 +633,6 @@ public:
const Store* store,
ThreadPool* thread_pool,
const std::unordered_map<std::string, field>& search_schema,
- const std::map<std::string, field>& facet_schema,
const std::unordered_map<std::string, field>& sort_schema,
const std::vector<char>& symbols_to_index, const std::vector<char>& token_separators);
@@ -679,7 +676,6 @@ public:
static void compute_token_offsets_facets(index_record& record,
const std::unordered_map<std::string, field>& search_schema,
- const std::map<std::string, field>& facet_schema,
const std::vector<char>& local_token_separators,
const std::vector<char>& local_symbols_to_index);
@@ -731,7 +727,6 @@ public:
const size_t batch_start_index, const size_t batch_size,
const std::string & default_sorting_field,
const std::unordered_map<std::string, field> & search_schema,
- const std::map<std::string, field> & facet_schema,
const std::string& fallback_field_type,
const std::vector<char>& token_separators,
const std::vector<char>& symbols_to_index);
@@ -740,7 +735,6 @@ public:
std::vector<index_record> & iter_batch,
const std::string & default_sorting_field,
const std::unordered_map<std::string, field> & search_schema,
- const std::map<std::string, field> & facet_schema,
const std::string& fallback_field_type,
const std::vector<char>& token_separators,
const std::vector<char>& symbols_to_index);
@@ -768,7 +762,6 @@ public:
static Option<uint32_t> validate_index_in_memory(nlohmann::json &document, uint32_t seq_id,
const std::string & default_sorting_field,
const std::unordered_map<std::string, field> & search_schema,
- const std::map<std::string, field> & facet_schema,
const index_operation_t op,
const std::string& fallback_field_type,
const DIRTY_VALUES& dirty_values);
diff --git a/src/collection.cpp b/src/collection.cpp
index 8a82a7c8..cc6349e9 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -354,7 +354,7 @@ Option<uint32_t> Collection::index_in_memory(nlohmann::json &document, uint32_t
std::unique_lock lock(mutex);
Option<uint32_t> validation_op = Index::validate_index_in_memory(document, seq_id, default_sorting_field,
- search_schema, facet_schema, op,
+ search_schema, op,
fallback_field_type, dirty_values);
if(!validation_op.ok()) {
@@ -362,11 +362,11 @@ Option<uint32_t> Collection::index_in_memory(nlohmann::json &document, uint32_t
}
index_record rec(0, seq_id, document, op, dirty_values);
- Index::compute_token_offsets_facets(rec, search_schema, facet_schema, token_separators, symbols_to_index);
+ Index::compute_token_offsets_facets(rec, search_schema, token_separators, symbols_to_index);
std::vector<index_record> index_batch;
index_batch.emplace_back(std::move(rec));
- Index::batch_memory_index(index, index_batch, default_sorting_field, search_schema, facet_schema,
+ Index::batch_memory_index(index, index_batch, default_sorting_field, search_schema,
fallback_field_type, token_separators, symbols_to_index);
num_documents += 1;
@@ -376,7 +376,7 @@ Option<uint32_t> Collection::index_in_memory(nlohmann::json &document, uint32_t
size_t Collection::batch_index_in_memory(std::vector<index_record>& index_records) {
std::unique_lock lock(mutex);
size_t num_indexed = Index::batch_memory_index(index, index_records, default_sorting_field,
- search_schema, facet_schema, fallback_field_type,
+ search_schema, fallback_field_type,
token_separators, symbols_to_index);
num_documents += num_indexed;
return num_indexed;
@@ -804,7 +804,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
// validate facet fields
for(const std::string & field_name: facet_fields) {
- if(facet_schema.count(field_name) == 0) {
+ if(search_schema.count(field_name) == 0 || !search_schema.at(field_name).facet) {
std::string error = "Could not find a facet field named `" + field_name + "` in the schema.";
return Option<nlohmann::json>(404, error);
}
@@ -845,7 +845,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
return Option<nlohmann::json>(400, error);
}
- if(facet_schema.count(facet_query.field_name) == 0) {
+ if(search_schema.count(facet_query.field_name) == 0 || !search_schema.at(facet_query.field_name).facet) {
std::string error = "Could not find a facet field named `" + facet_query.field_name + "` in the schema.";
return Option<nlohmann::json>(404, error);
}
@@ -1664,52 +1664,52 @@ bool Collection::facet_value_to_string(const facet &a_facet, const facet_count_t
}
LOG(ERROR) << "Could not find field " << a_facet.field_name << " in document during faceting.";
- LOG(ERROR) << "Facet field type: " << facet_schema.at(a_facet.field_name).type;
+ LOG(ERROR) << "Facet field type: " << search_schema.at(a_facet.field_name).type;
LOG(ERROR) << "Actual document: " << document;
return false;
}
- if(facet_schema.at(a_facet.field_name).is_array()) {
+ if(search_schema.at(a_facet.field_name).is_array()) {
size_t array_sz = document[a_facet.field_name].size();
if(facet_count.array_pos >= array_sz) {
LOG(ERROR) << "Facet field array size " << array_sz << " lesser than array pos " << facet_count.array_pos
<< " for facet field " << a_facet.field_name;
- LOG(ERROR) << "Facet field type: " << facet_schema.at(a_facet.field_name).type;
+ LOG(ERROR) << "Facet field type: " << search_schema.at(a_facet.field_name).type;
LOG(ERROR) << "Actual document: " << document;
return false;
}
}
- if(facet_schema.at(a_facet.field_name).type == field_types::STRING) {
+ if(search_schema.at(a_facet.field_name).type == field_types::STRING) {
value = document[a_facet.field_name];
- } else if(facet_schema.at(a_facet.field_name).type == field_types::STRING_ARRAY) {
+ } else if(search_schema.at(a_facet.field_name).type == field_types::STRING_ARRAY) {
value = document[a_facet.field_name][facet_count.array_pos];
- } else if(facet_schema.at(a_facet.field_name).type == field_types::INT32) {
+ } else if(search_schema.at(a_facet.field_name).type == field_types::INT32) {
int32_t raw_val = document[a_facet.field_name].get<int32_t>();
value = std::to_string(raw_val);
- } else if(facet_schema.at(a_facet.field_name).type == field_types::INT32_ARRAY) {
+ } else if(search_schema.at(a_facet.field_name).type == field_types::INT32_ARRAY) {
int32_t raw_val = document[a_facet.field_name][facet_count.array_pos].get<int32_t>();
value = std::to_string(raw_val);
- } else if(facet_schema.at(a_facet.field_name).type == field_types::INT64) {
+ } else if(search_schema.at(a_facet.field_name).type == field_types::INT64) {
int64_t raw_val = document[a_facet.field_name].get<int64_t>();
value = std::to_string(raw_val);
- } else if(facet_schema.at(a_facet.field_name).type == field_types::INT64_ARRAY) {
+ } else if(search_schema.at(a_facet.field_name).type == field_types::INT64_ARRAY) {
int64_t raw_val = document[a_facet.field_name][facet_count.array_pos].get<int64_t>();
value = std::to_string(raw_val);
- } else if(facet_schema.at(a_facet.field_name).type == field_types::FLOAT) {
+ } else if(search_schema.at(a_facet.field_name).type == field_types::FLOAT) {
float raw_val = document[a_facet.field_name].get<float>();
value = StringUtils::float_to_str(raw_val);
if(value != "0") {
value.erase ( value.find_last_not_of('0') + 1, std::string::npos ); // remove trailing zeros
}
- } else if(facet_schema.at(a_facet.field_name).type == field_types::FLOAT_ARRAY) {
+ } else if(search_schema.at(a_facet.field_name).type == field_types::FLOAT_ARRAY) {
float raw_val = document[a_facet.field_name][facet_count.array_pos].get<float>();
value = StringUtils::float_to_str(raw_val);
value.erase ( value.find_last_not_of('0') + 1, std::string::npos ); // remove trailing zeros
- } else if(facet_schema.at(a_facet.field_name).type == field_types::BOOL) {
+ } else if(search_schema.at(a_facet.field_name).type == field_types::BOOL) {
value = std::to_string(document[a_facet.field_name].get<bool>());
value = (value == "1") ? "true" : "false";
- } else if(facet_schema.at(a_facet.field_name).type == field_types::BOOL_ARRAY) {
+ } else if(search_schema.at(a_facet.field_name).type == field_types::BOOL_ARRAY) {
value = std::to_string(document[a_facet.field_name][facet_count.array_pos].get<bool>());
value = (value == "1") ? "true" : "false";
}
@@ -2282,8 +2282,10 @@ std::vector<std::string> Collection::get_facet_fields() {
std::shared_lock lock(mutex);
std::vector<std::string> facet_fields_copy;
- for(auto it = facet_schema.begin(); it != facet_schema.end(); ++it) {
- facet_fields_copy.push_back(it->first);
+ for(auto it = search_schema.begin(); it != search_schema.end(); ++it) {
+ if(it->second.facet) {
+ facet_fields_copy.push_back(it->first);
+ }
}
return facet_fields_copy;
@@ -2701,10 +2703,6 @@ Option<bool> Collection::check_and_update_schema(nlohmann::json& document, const
}
search_schema.emplace(new_field.name, new_field);
- if(new_field.is_facet()) {
- facet_schema.emplace(new_field.name, new_field);
- }
-
fields.emplace_back(new_field);
}
@@ -2760,10 +2758,6 @@ Index* Collection::init_index() {
search_schema.emplace(field.name, field);
- if(field.is_facet()) {
- facet_schema.emplace(field.name, field);
- }
-
if(field.is_sortable()) {
sort_schema.emplace(field.name, field);
}
@@ -2773,7 +2767,7 @@ Index* Collection::init_index() {
collection_id,
store,
CollectionManager::get_instance().get_thread_pool(),
- search_schema, facet_schema, sort_schema,
+ search_schema, sort_schema,
symbols_to_index, token_separators);
}
diff --git a/src/index.cpp b/src/index.cpp
index 154706e7..0a1b2c9a 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -40,10 +40,10 @@ spp::sparse_hash_map<uint32_t, int64_t> Index::str_sentinel_value;
Index::Index(const std::string& name, const uint32_t collection_id, const Store* store, ThreadPool* thread_pool,
const std::unordered_map<std::string, field> & search_schema,
- const std::map<std::string, field>& facet_schema, const std::unordered_map<std::string, field>& sort_schema,
+ const std::unordered_map<std::string, field>& sort_schema,
const std::vector<char>& symbols_to_index, const std::vector<char>& token_separators):
name(name), collection_id(collection_id), store(store), thread_pool(thread_pool),
- search_schema(search_schema), facet_schema(facet_schema), sort_schema(sort_schema),
+ search_schema(search_schema), sort_schema(sort_schema),
seq_ids(new id_list_t(256)), symbols_to_index(symbols_to_index), token_separators(token_separators) {
for(const auto & fname_field: search_schema) {
@@ -66,6 +66,15 @@ Index::Index(const std::string& name, const uint32_t collection_id, const Store*
numerical_index.emplace(fname_field.first, num_tree);
}
+ if(fname_field.second.facet) {
+ array_mapped_facet_t facet_array;
+ for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {
+ facet_array[i] = new facet_map_t();
+ }
+
+ facet_index_v3.emplace(fname_field.first, facet_array);
+ }
+
// initialize for non-string facet fields
if(fname_field.second.facet && !fname_field.second.is_string()) {
art_tree *ft = new art_tree;
@@ -94,15 +103,6 @@ Index::Index(const std::string& name, const uint32_t collection_id, const Store*
}
}
- for(const auto& pair: facet_schema) {
- array_mapped_facet_t facet_array;
- for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {
- facet_array[i] = new facet_map_t();
- }
-
- facet_index_v3.emplace(pair.first, facet_array);
- }
-
num_documents = 0;
}
@@ -208,7 +208,6 @@ int64_t Index::float_to_in64_t(float f) {
void Index::compute_token_offsets_facets(index_record& record,
const std::unordered_map<std::string, field>& search_schema,
- const std::map<std::string, field>& facet_schema,
const std::vector<char>& local_token_separators,
const std::vector<char>& local_symbols_to_index) {
@@ -222,7 +221,7 @@ void Index::compute_token_offsets_facets(index_record& record,
offsets_facet_hashes_t offset_facet_hashes;
- bool is_facet = (facet_schema.count(field_name) != 0);
+ bool is_facet = search_schema.at(field_name).facet;
// non-string, non-geo faceted field should be indexed as faceted string field as well
if(field_pair.second.facet && !field_pair.second.is_string() && !field_pair.second.is_geopoint()) {
@@ -291,7 +290,6 @@ void Index::compute_token_offsets_facets(index_record& record,
Option<uint32_t> Index::validate_index_in_memory(nlohmann::json& document, uint32_t seq_id,
const std::string & default_sorting_field,
const std::unordered_map<std::string, field> & search_schema,
- const std::map<std::string, field> & facet_schema,
const index_operation_t op,
const std::string& fallback_field_type,
const DIRTY_VALUES& dirty_values) {
@@ -443,7 +441,6 @@ void Index::validate_and_preprocess(Index *index, std::vector<index_record>& ite
const size_t batch_start_index, const size_t batch_size,
const std::string& default_sorting_field,
const std::unordered_map<std::string, field>& search_schema,
- const std::map<std::string, field>& facet_schema,
const std::string& fallback_field_type,
const std::vector<char>& token_separators,
const std::vector<char>& symbols_to_index) {
@@ -465,7 +462,7 @@ void Index::validate_and_preprocess(Index *index, std::vector<index_record>& ite
Option<uint32_t> validation_op = validate_index_in_memory(index_rec.doc, index_rec.seq_id,
default_sorting_field,
- search_schema, facet_schema,
+ search_schema,
index_rec.operation,
fallback_field_type,
index_rec.dirty_values);
@@ -482,7 +479,7 @@ void Index::validate_and_preprocess(Index *index, std::vector<index_record>& ite
scrub_reindex_doc(search_schema, index_rec.doc, index_rec.del_doc, index_rec.old_doc);
}
- compute_token_offsets_facets(index_rec, search_schema, facet_schema, token_separators, symbols_to_index);
+ compute_token_offsets_facets(index_rec, search_schema, token_separators, symbols_to_index);
int64_t points = 0;
@@ -514,7 +511,6 @@ void Index::validate_and_preprocess(Index *index, std::vector<index_record>& ite
size_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_batch,
const std::string & default_sorting_field,
const std::unordered_map<std::string, field> & search_schema,
- const std::map<std::string, field> & facet_schema,
const std::string& fallback_field_type,
const std::vector<char>& token_separators,
const std::vector<char>& symbols_to_index) {
@@ -543,8 +539,7 @@ size_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_b
index->thread_pool->enqueue([&, batch_index, batch_len]() {
validate_and_preprocess(index, iter_batch, batch_index, batch_len, default_sorting_field, search_schema,
- facet_schema, fallback_field_type,
- token_separators, symbols_to_index);
+ fallback_field_type, token_separators, symbols_to_index);
std::unique_lock<std::mutex> lock(m_process);
num_processed++;
@@ -2921,7 +2916,7 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&
facet_infos[findex].use_facet_query = false;
- const field &facet_field = facet_schema.at(a_facet.field_name);
+ const field &facet_field = search_schema.at(a_facet.field_name);
facet_infos[findex].facet_field = facet_field;
facet_infos[findex].should_compute_stats = (facet_field.type != field_types::STRING &&
@@ -3944,8 +3939,6 @@ void Index::refresh_schemas(const std::vector<field>& new_fields) {
}
if(new_field.is_facet()) {
- facet_schema.emplace(new_field.name, new_field);
-
array_mapped_facet_t facet_array;
for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {
facet_array[i] = new facet_map_t();
diff --git a/test/index_test.cpp b/test/index_test.cpp
index 554cd31b..5bff3b81 100644
--- a/test/index_test.cpp
+++ b/test/index_test.cpp
@@ -12,7 +12,7 @@ TEST(IndexTest, ScrubReindexDoc) {
ThreadPool pool(4);
- Index index("index", 1, nullptr, &pool, search_schema, {}, {}, {}, {});
+ Index index("index", 1, nullptr, &pool, search_schema, {}, {}, {});
nlohmann::json old_doc;
old_doc["id"] = "1";
old_doc["title"] = "One more thing.";