Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Field wise indexing.
Kishore Nallan14f4c3e834a2
diff --git a/include/art.h b/include/art.hindex e87117cf..04739e65 100644--- a/include/art.h+++ b/include/art.h@@ -116,11 +116,16 @@ typedef struct {* Represents a document to be indexed.* `offsets` refer to the index locations where a token appeared in the document*/-typedef struct {- int64_t score;- uint32_t id;- std::vector<uint32_t> offsets;-} art_document;+struct art_document {+ const uint32_t id;+ const int64_t score;+ const std::vector<uint32_t> offsets;++ art_document(const uint32_t id, const int64_t score, const std::vector<uint32_t>& offsets):+ id(id), score(score), offsets(offsets) {++ }+};enum token_ordering {NOT_SET,@@ -188,6 +193,10 @@ inline uint64_t art_size(art_tree *t) {*/void* art_insert(art_tree *t, const unsigned char *key, int key_len, art_document* document);+/* Insert multiple docs sharing the same key */+void* art_inserts(art_tree *t, const unsigned char *key, int key_len, const int64_t docs_max_score,+ std::vector<art_document>& documents);+/*** Deletes a value from the ART tree* @arg t The treediff --git a/include/index.h b/include/index.hindex 7d8f83b8..2b55deb7 100644--- a/include/index.h+++ b/include/index.h@@ -355,6 +355,7 @@ struct index_record {// pre-processed data primed for indexingstd::unordered_map<std::string, offsets_facet_hashes_t> field_index;+ int64_t points;Option<bool> indexed; // indicates if the indexing operation was a success@@ -681,10 +682,6 @@ public:Option<uint32_t> remove(const uint32_t seq_id, const nlohmann::json & document, const bool is_update);- Option<uint32_t> index_in_memory(const index_record& record, uint32_t seq_id,- const std::string& default_sorting_field,- const bool is_update);-static void validate_and_preprocess(Index *index, std::vector<index_record>& iter_batch,const size_t batch_start_index, const size_t batch_size,const std::string & default_sorting_field,@@ -703,6 +700,11 @@ public:const std::vector<char>& token_separators,const std::vector<char>& symbols_to_index);+ void index_field_in_memory(const field& afield, std::vector<index_record>& iter_batch);++ template<class T>+ void iterate_and_index_numerical_field(std::vector<index_record>& iter_batch, const field& afield, T func);+//static bool is_point_in_polygon(const Geofence& poly, const GeoCoord& point);//static double transform_for_180th_meridian(Geofence& poly);@@ -753,3 +755,20 @@ public:std::vector<facet_info_t>& facet_infos) const;};+template<class T>+void Index::iterate_and_index_numerical_field(std::vector<index_record>& iter_batch, const field& afield, T func) {+ for(const auto& record: iter_batch) {+ if(!record.indexed.ok()) {+ continue;+ }++ const auto& document = record.doc;+ const auto seq_id = record.seq_id;++ if (document.count(afield.name) == 0 || !afield.index) {+ continue;+ }++ func(record, seq_id);+ }+}diff --git a/src/art.cpp b/src/art.cppindex cba9cb04..40017d0c 100644--- a/src/art.cpp+++ b/src/art.cpp@@ -421,7 +421,6 @@ static void add_document_to_leaf(art_document *document, art_leaf *leaf) {if(document->score == USE_FREQUENCY_SCORE) {leaf->max_score = posting_t::num_ids(leaf->values);- document->score = leaf->max_score;}}@@ -609,11 +608,17 @@ static int prefix_mismatch(const art_node *n, const unsigned char *key, int key_return idx;}-static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *key, uint32_t key_len,- art_document *document, int depth, std::list<art_node*>& path, int *old) {+static void* recursive_insert(art_node* n, art_node** ref, const unsigned char* key, uint32_t key_len,+ const int64_t docs_max_score, std::vector<art_document>& documents, int depth,+ std::list<art_node*>& path, int* old) {// If we are at a NULL node, inject a leafif (!n) {- *ref = (art_node*)SET_LEAF(make_leaf(key, key_len, document));+ art_leaf* new_leaf = make_leaf(key, key_len, &documents[0]);+ for(size_t i = 1; i < documents.size(); i++) {+ add_document_to_leaf(&documents[i], new_leaf);+ }++ *ref = (art_node*)SET_LEAF(new_leaf);return NULL;}@@ -624,7 +629,9 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *// Check if we are updating an existing valueif (!leaf_matches(l, key, key_len, depth)) {*old = 1;- add_document_to_leaf(document, l);+ for(size_t i = 0; i < documents.size(); i++) {+ add_document_to_leaf(&documents[i], l);+ }return l->values;}@@ -632,12 +639,16 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *art_node4 *new_n = (art_node4*)alloc_node(NODE4);// Create a new leaf- art_leaf *l2 = make_leaf(key, key_len, document);+ art_leaf *l2 = make_leaf(key, key_len, &documents[0]);uint32_t longest_prefix = longest_common_prefix(l, l2, depth);new_n->n.partial_len = longest_prefix;memcpy(new_n->n.partial, key+depth, min(MAX_PREFIX_LEN, longest_prefix));+ for(size_t i = 1; i < documents.size(); i++) {+ add_document_to_leaf(&documents[i], l2);+ }+// Add the leafs to the new node4*ref = (art_node*)new_n;add_child4(new_n, ref, l->key[depth+longest_prefix], SET_LEAF(l));@@ -645,8 +656,8 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *return NULL;}- if(document->score != USE_FREQUENCY_SCORE) {- n->max_score = MAX(n->max_score, document->score);+ if(docs_max_score != USE_FREQUENCY_SCORE) {+ n->max_score = MAX(n->max_score, docs_max_score);}// Check if given node has a prefix@@ -679,7 +690,11 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *}// Insert the new leaf- art_leaf *l = make_leaf(key, key_len, document);+ art_leaf *l = make_leaf(key, key_len, &documents[0]);+ for(size_t i = 1; i < documents.size(); i++) {+ add_document_to_leaf(&documents[i], l);+ }+add_child4(new_n, ref, key[depth+prefix_diff], SET_LEAF(l));path.push_back(*ref);return NULL;@@ -690,11 +705,15 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *// Find a child to recurse toart_node **child = find_child(n, key[depth]);if (child) {- return recursive_insert(*child, child, key, key_len, document, depth + 1, path, old);+ return recursive_insert(*child, child, key, key_len, docs_max_score, documents, depth + 1, path, old);}// No child, node goes within us- art_leaf *l = make_leaf(key, key_len, document);+ art_leaf *l = make_leaf(key, key_len, &documents[0]);+ for(size_t i = 1; i < documents.size(); i++) {+ add_document_to_leaf(&documents[i], l);+ }+add_child(n, ref, key[depth], SET_LEAF(l));path.push_back(*ref);return NULL;@@ -710,16 +729,22 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char ** the old value pointer is returned.*/void* art_insert(art_tree *t, const unsigned char *key, int key_len, art_document* document) {+ std::vector<art_document> documents = {*document};+ return art_inserts(t, key, key_len, document->score, documents);+}++void* art_inserts(art_tree *t, const unsigned char *key, int key_len, const int64_t docs_max_score,+ std::vector<art_document>& documents) {int old_val = 0;std::list<art_node*> path;- bool frequency_based_ordering = (document->score == USE_FREQUENCY_SCORE);- void *old = recursive_insert(t->root, &t->root, key, key_len, document, 0, path, &old_val);+ bool frequency_based_ordering = (docs_max_score == USE_FREQUENCY_SCORE);+ void *old = recursive_insert(t->root, &t->root, key, key_len, docs_max_score, documents, 0, path, &old_val);if (!old_val) t->size++;if(frequency_based_ordering) {for(art_node* n: path) {- n->max_score = MAX(n->max_score, document->score);+ n->max_score = MAX(n->max_score, docs_max_score);}}diff --git a/src/collection.cpp b/src/collection.cppindex a28775d3..1dd85f3a 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -368,7 +368,11 @@ Option<uint32_t> Collection::index_in_memory(nlohmann::json &document, uint32_tindex_record rec(0, seq_id, document, op, dirty_values);Index::compute_token_offsets_facets(rec, search_schema, facet_schema, token_separators, symbols_to_index);- index->index_in_memory(rec, seq_id, default_sorting_field, op);++ std::vector<index_record> index_batch;+ index_batch.emplace_back(std::move(rec));+ Index::batch_memory_index(index, index_batch, default_sorting_field, search_schema, facet_schema,+ fallback_field_type, token_separators, symbols_to_index);num_documents += 1;return Option<>(200);@@ -2429,6 +2433,10 @@ Index* Collection::init_index() {continue;}+ if(field.name == ".*") {+ continue;+ }+search_schema.emplace(field.name, field);if(field.is_facet()) {diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 7dae91d3..e3307c05 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -161,7 +161,13 @@ Option<bool> CollectionManager::load(const size_t collection_batch_size, const sauto captured_store = store;loading_pool.enqueue([captured_store, num_collections, collection_meta, document_batch_size,&m_process, &cv_process, &num_processed, &next_coll_id_status, quit = quit]() {++ //auto begin = std::chrono::high_resolution_clock::now();Option<bool> res = load_collection(collection_meta, document_batch_size, next_coll_id_status, *quit);+ /*long long int timeMillis =+ std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();+ LOG(INFO) << "Time taken for indexing: " << timeMillis << "ms";*/+if(!res.ok()) {LOG(ERROR) << "Error while loading collection. " << res.error();LOG(ERROR) << "Typesense is quitting.";diff --git a/src/index.cpp b/src/index.cppindex 22341a8f..403f020c 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -18,6 +18,7 @@#include <s2/s2builder.h>#include <posting.h>#include <thread_local_vars.h>+#include <unordered_set>#include "logger.h"#define RETURN_CIRCUIT_BREAKER if(std::chrono::duration_cast<std::chrono::milliseconds>(\@@ -252,182 +253,6 @@ void Index::compute_token_offsets_facets(index_record& record,}}-Option<uint32_t> Index::index_in_memory(const index_record& record, uint32_t seq_id,- const std::string & default_sorting_field,- const bool is_update) {- std::unique_lock lock(mutex);-- int64_t points = 0;- auto& document = record.doc;-- if(document.count(default_sorting_field) == 0) {- auto default_sorting_field_it = sort_index.find(default_sorting_field);- if(default_sorting_field_it != sort_index.end()) {- auto seq_id_it = default_sorting_field_it->second->find(seq_id);- if(seq_id_it != default_sorting_field_it->second->end()) {- points = seq_id_it->second;- } else {- points = INT64_MIN;- }- } else {- points = INT64_MIN;- }- } else {- points = get_points_from_doc(document, default_sorting_field);- }-- if(!is_update) {- // for updates, the seq_id will already exist- seq_ids.append(seq_id);- }-- // assumes that validation has already been done- for(const auto& field_pair: search_schema) {- const std::string & field_name = field_pair.first;-- if(document.count(field_name) == 0 || !field_pair.second.index) {- continue;- }-- bool is_facet = (facet_schema.count(field_name) != 0);-- // non-string, non-geo faceted field should be indexed as faceted string field as well- if(field_pair.second.facet && !field_pair.second.is_string() && !field_pair.second.is_geopoint()) {- auto field_index_it = record.field_index.find(field_name);- if(field_index_it == record.field_index.end()) {- continue;- }-- art_tree *t = search_index.at(field_pair.second.faceted_name());-- index_strings_field(points, t, seq_id, is_facet, field_pair.second,- field_index_it->second.offsets,- field_index_it->second.facet_hashes);- }-- if(field_pair.second.is_string()) {- auto field_index_it = record.field_index.find(field_name);- if(field_index_it == record.field_index.end()) {- continue;- }-- art_tree *t = search_index.at(field_name);- index_strings_field(points, t, seq_id, is_facet, field_pair.second,- field_index_it->second.offsets,- field_index_it->second.facet_hashes);- } else if(field_pair.second.type == field_types::INT32) {- auto num_tree = numerical_index.at(field_name);- int32_t value = document[field_name].get<int32_t>();- num_tree->insert(value, seq_id);- } else if(field_pair.second.type == field_types::INT64) {- auto num_tree = numerical_index.at(field_name);- int64_t value = document[field_name].get<int64_t>();- num_tree->insert(value, seq_id);- } else if(field_pair.second.type == field_types::FLOAT) {- auto num_tree = numerical_index.at(field_name);- float fvalue = document[field_name].get<float>();- int64_t value = float_to_in64_t(fvalue);- num_tree->insert(value, seq_id);- } else if(field_pair.second.type == field_types::BOOL) {- auto num_tree = numerical_index.at(field_name);- bool value = document[field_name];- num_tree->insert(value, seq_id);- } else if(field_pair.second.type == field_types::GEOPOINT) {- const std::vector<double>& latlong = document[field_name];- auto geo_index = geopoint_index.at(field_name);-- S2RegionTermIndexer::Options options;- options.set_index_contains_points_only(true);- S2RegionTermIndexer indexer(options);- S2Point point = S2LatLng::FromDegrees(latlong[0], latlong[1]).ToPoint();-- for(const auto& term: indexer.GetIndexTerms(point, "")) {- (*geo_index)[term].push_back(seq_id);- }- } else if(field_pair.second.type == field_types::GEOPOINT_ARRAY) {- const std::vector<std::vector<double>>& latlongs = document[field_name];- auto geo_index = geopoint_index.at(field_name);- S2RegionTermIndexer::Options options;- options.set_index_contains_points_only(true);- S2RegionTermIndexer indexer(options);-- int64_t* packed_latlongs = new int64_t[latlongs.size() + 1];- packed_latlongs[0] = latlongs.size();-- for(size_t li = 0; li < latlongs.size(); li++) {- auto& latlong = latlongs[li];- S2Point point = S2LatLng::FromDegrees(latlong[0], latlong[1]).ToPoint();- std::set<std::string> terms;- for(const auto& term: indexer.GetIndexTerms(point, "")) {- terms.insert(term);- }-- for(const auto& term: terms) {- (*geo_index)[term].push_back(seq_id);- }-- int64_t packed_latlong = GeoPoint::pack_lat_lng(latlong[0], latlong[1]);- packed_latlongs[li + 1] = packed_latlong;- }-- geo_array_index.at(field_name)->emplace(seq_id, packed_latlongs);- }-- else if(field_pair.second.is_array()) {- // all other numerical arrays- auto num_tree = numerical_index.at(field_name);-- for(size_t arr_i = 0; arr_i < document[field_name].size(); arr_i++) {- const auto& arr_value = document[field_name][arr_i];-- if(field_pair.second.type == field_types::INT32_ARRAY) {- const int32_t value = arr_value;- num_tree->insert(value, seq_id);- }-- else if(field_pair.second.type == field_types::INT64_ARRAY) {- const int64_t value = arr_value;- num_tree->insert(value, seq_id);- }-- else if(field_pair.second.type == field_types::FLOAT_ARRAY) {- const float fvalue = arr_value;- int64_t value = float_to_in64_t(fvalue);- num_tree->insert(value, seq_id);- }-- else if(field_pair.second.type == field_types::BOOL_ARRAY) {- const bool value = document[field_name][arr_i];- num_tree->insert(int64_t(value), seq_id);- }- }- }-- // add numerical values automatically into sort index- if(field_pair.second.type == field_types::INT32 || field_pair.second.type == field_types::INT64 ||- field_pair.second.type == field_types::FLOAT || field_pair.second.type == field_types::BOOL ||- field_pair.second.type == field_types::GEOPOINT) {- spp::sparse_hash_map<uint32_t, int64_t> *doc_to_score = sort_index.at(field_pair.first);-- if(field_pair.second.is_integer() ) {- doc_to_score->emplace(seq_id, document[field_pair.first].get<int64_t>());- } else if(field_pair.second.is_float()) {- int64_t ifloat = float_to_in64_t(document[field_pair.first].get<float>());- doc_to_score->emplace(seq_id, ifloat);- } else if(field_pair.second.is_bool()) {- doc_to_score->emplace(seq_id, (int64_t) document[field_pair.first].get<bool>());- } else if(field_pair.second.is_geopoint()) {- const std::vector<double>& latlong = document[field_pair.first];- int64_t lat_lng = GeoPoint::pack_lat_lng(latlong[0], latlong[1]);- doc_to_score->emplace(seq_id, lat_lng);- }- }- }-- num_documents += 1;- return Option<>(201);-}-Option<uint32_t> Index::validate_index_in_memory(nlohmann::json& document, uint32_t seq_id,const std::string & default_sorting_field,const std::unordered_map<std::string, field> & search_schema,@@ -628,6 +453,27 @@ void Index::validate_and_preprocess(Index *index, std::vector<index_record>& ite}compute_token_offsets_facets(index_rec, search_schema, facet_schema, token_separators, symbols_to_index);++ int64_t points = 0;++ if(index_rec.doc.count(default_sorting_field) == 0) {+ auto default_sorting_field_it = index->sort_index.find(default_sorting_field);+ if(default_sorting_field_it != index->sort_index.end()) {+ auto seq_id_it = default_sorting_field_it->second->find(index_rec.seq_id);+ if(seq_id_it != default_sorting_field_it->second->end()) {+ points = seq_id_it->second;+ } else {+ points = INT64_MIN;+ }+ } else {+ points = INT64_MIN;+ }+ } else {+ points = get_points_from_doc(index_rec.doc, default_sorting_field);+ }++ index_rec.points = points;+ index_rec.index_success();}}@@ -674,51 +520,290 @@ size_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_bbatch_index += batch_len;}- std::unique_lock<std::mutex> lock_process(m_process);- cv_process.wait(lock_process, [&](){ return num_processed == num_queued; });+ {+ std::unique_lock<std::mutex> lock_process(m_process);+ cv_process.wait(lock_process, [&](){ return num_processed == num_queued; });+ }- Option<uint32_t> index_mem_op(0);+ std::unordered_set<std::string> found_fields;- for(auto& index_rec: iter_batch) {- if(index_rec.operation == DELETE) {- continue;+ for(size_t i = 0; i < iter_batch.size(); i++) {+ auto& index_rec = iter_batch[i];+ if(index_rec.is_update) {+ index->remove(index_rec.seq_id, index_rec.del_doc, index_rec.is_update);+ } else {+ num_indexed++;}- if(!index_rec.indexed.ok()) {- // some records could have been invalidated upstream+ for(const auto& kv: index_rec.doc.items()) {+ found_fields.insert(kv.key());+ }+ }++ num_queued = num_processed = 0;++ for(const auto& field_name: found_fields) {+ //LOG(INFO) << "field name: " << field_name;+ if(field_name != "id" && search_schema.count(field_name) == 0) {continue;}- if(index_rec.is_update) {- index->remove(index_rec.seq_id, index_rec.del_doc, index_rec.is_update);+ num_queued++;++ index->thread_pool->enqueue([&]() {+ const field& f = (field_name == "id") ?+ field("id", field_types::STRING, false) : search_schema.at(field_name);+ index->index_field_in_memory(f, iter_batch);+ std::unique_lock<std::mutex> lock(m_process);+ num_processed++;+ cv_process.notify_one();+ });+ }++ {+ std::unique_lock<std::mutex> lock_process(m_process);+ cv_process.wait(lock_process, [&](){ return num_processed == num_queued; });+ }++ return num_indexed;+}++void Index::index_field_in_memory(const field& afield, std::vector<index_record>& iter_batch) {+ // indexes a given field of all documents in the batch++ if(afield.name == "id") {+ for(const auto& record: iter_batch) {+ if(!record.is_update) {+ // for updates, the seq_id will already exist+ seq_ids.append(record.seq_id);+ }}- try {- index_mem_op = index->index_in_memory(index_rec, index_rec.seq_id, default_sorting_field, index_rec.is_update);- } catch(const std::exception& e) {- const std::string& error_msg = std::string("Fatal error during indexing: ") + e.what();- LOG(ERROR) << error_msg << ", document: " << index_rec.doc;- index_mem_op = Option<uint32_t>(500, error_msg);+ return;+ }++ // We have to handle both these edge cases:+ // a) `afield` might not exist in the document (optional field)+ // b) `afield` value could be empty++ // non-geo faceted field should be indexed as faceted string field as well+ bool non_string_facet_field = (afield.facet && !afield.is_geopoint());++ if(afield.is_string() || non_string_facet_field) {+ std::unordered_map<std::string, std::vector<art_document>> token_to_doc_offsets;+ int64_t max_score = INT64_MIN;++ for(const auto& record: iter_batch) {+ const auto& document = record.doc;+ const auto seq_id = record.seq_id;++ if(document.count(afield.name) == 0 || !afield.index) {+ continue;+ }++ auto field_index_it = record.field_index.find(afield.name);+ if(field_index_it == record.field_index.end()) {+ continue;+ }++ if(afield.facet) {+ facet_hash_values_t fhashvalues;+ fhashvalues.length = field_index_it->second.facet_hashes.size();+ fhashvalues.hashes = new uint64_t[field_index_it->second.facet_hashes.size()];++ for(size_t i = 0; i < field_index_it->second.facet_hashes.size(); i++) {+ fhashvalues.hashes[i] = field_index_it->second.facet_hashes[i];+ }++ facet_index_v3[afield.name][seq_id % ARRAY_FACET_DIM]->emplace(seq_id, std::move(fhashvalues));+ }++ if(record.points > max_score) {+ max_score = record.points;+ }++ for(auto &token_offsets: field_index_it->second.offsets) {+ token_to_doc_offsets[token_offsets.first].emplace_back(seq_id, record.points, token_offsets.second);+ }}- index_rec.index_success();+ auto tree_it = search_index.find(afield.faceted_name());+ if(tree_it == search_index.end()) {+ return;+ }- if(!index_rec.is_update) {- num_indexed++;+ art_tree *t = tree_it->second;++ for(auto& token_to_doc: token_to_doc_offsets) {+ std::vector<art_document>& documents = token_to_doc.second;+ const std::string& token = token_to_doc.first;++ const auto *key = (const unsigned char *) token.c_str();+ int key_len = (int) token.length() + 1; // for the terminating \0 char++ //LOG(INFO) << "key: " << key << ", art_doc.id: " << art_doc.id;+ art_inserts(t, key, key_len, max_score, documents);}}- return num_indexed;+ if(!afield.is_string()) {+ if (afield.type == field_types::INT32) {+ auto num_tree = numerical_index.at(afield.name);+ iterate_and_index_numerical_field(iter_batch, afield, [&afield, num_tree]+ (const index_record& record, uint32_t seq_id) {+ int32_t value = record.doc[afield.name].get<int32_t>();+ num_tree->insert(value, seq_id);+ });+ }++ else if(afield.type == field_types::INT64) {+ auto num_tree = numerical_index.at(afield.name);+ iterate_and_index_numerical_field(iter_batch, afield, [&afield, num_tree]+ (const index_record& record, uint32_t seq_id) {+ int64_t value = record.doc[afield.name].get<int64_t>();+ num_tree->insert(value, seq_id);+ });+ }++ else if(afield.type == field_types::FLOAT) {+ auto num_tree = numerical_index.at(afield.name);+ iterate_and_index_numerical_field(iter_batch, afield, [&afield, num_tree]+ (const index_record& record, uint32_t seq_id) {+ float fvalue = record.doc[afield.name].get<float>();+ int64_t value = float_to_in64_t(fvalue);+ num_tree->insert(value, seq_id);+ });+ } else if(afield.type == field_types::BOOL) {+ auto num_tree = numerical_index.at(afield.name);+ iterate_and_index_numerical_field(iter_batch, afield, [&afield, num_tree]+ (const index_record& record, uint32_t seq_id) {+ bool value = record.doc[afield.name].get<bool>();+ num_tree->insert(value, seq_id);+ });+ } else if(afield.type == field_types::GEOPOINT) {+ auto geo_index = geopoint_index.at(afield.name);++ iterate_and_index_numerical_field(iter_batch, afield, [&afield, geo_index]+ (const index_record& record, uint32_t seq_id) {+ const std::vector<double>& latlong = record.doc[afield.name];++ S2RegionTermIndexer::Options options;+ options.set_index_contains_points_only(true);+ S2RegionTermIndexer indexer(options);+ S2Point point = S2LatLng::FromDegrees(latlong[0], latlong[1]).ToPoint();++ for(const auto& term: indexer.GetIndexTerms(point, "")) {+ (*geo_index)[term].push_back(seq_id);+ }+ });+ } else if(afield.type == field_types::GEOPOINT_ARRAY) {+ auto geo_index = geopoint_index.at(afield.name);++ iterate_and_index_numerical_field(iter_batch, afield,+ [&afield, &geo_array_index=geo_array_index, geo_index](const index_record& record, uint32_t seq_id) {++ const std::vector<std::vector<double>>& latlongs = record.doc[afield.name];+ S2RegionTermIndexer::Options options;+ options.set_index_contains_points_only(true);+ S2RegionTermIndexer indexer(options);++ int64_t* packed_latlongs = new int64_t[latlongs.size() + 1];+ packed_latlongs[0] = latlongs.size();++ for(size_t li = 0; li < latlongs.size(); li++) {+ auto& latlong = latlongs[li];+ S2Point point = S2LatLng::FromDegrees(latlong[0], latlong[1]).ToPoint();+ std::set<std::string> terms;+ for(const auto& term: indexer.GetIndexTerms(point, "")) {+ terms.insert(term);+ }++ for(const auto& term: terms) {+ (*geo_index)[term].push_back(seq_id);+ }++ int64_t packed_latlong = GeoPoint::pack_lat_lng(latlong[0], latlong[1]);+ packed_latlongs[li + 1] = packed_latlong;+ }++ geo_array_index.at(afield.name)->emplace(seq_id, packed_latlongs);+ });+ } else if(afield.is_array()) {+ // all other numerical arrays+ auto num_tree = numerical_index.at(afield.name);+ iterate_and_index_numerical_field(iter_batch, afield, [&afield, num_tree]+ (const index_record& record, uint32_t seq_id) {+ for(size_t arr_i = 0; arr_i < record.doc[afield.name].size(); arr_i++) {+ const auto& arr_value = record.doc[afield.name][arr_i];++ if(afield.type == field_types::INT32_ARRAY) {+ const int32_t value = arr_value;+ num_tree->insert(value, seq_id);+ }++ else if(afield.type == field_types::INT64_ARRAY) {+ const int64_t value = arr_value;+ num_tree->insert(value, seq_id);+ }++ else if(afield.type == field_types::FLOAT_ARRAY) {+ const float fvalue = arr_value;+ int64_t value = float_to_in64_t(fvalue);+ num_tree->insert(value, seq_id);+ }++ else if(afield.type == field_types::BOOL_ARRAY) {+ const bool value = record.doc[afield.name][arr_i];+ num_tree->insert(int64_t(value), seq_id);+ }+ }+ });+ }++ // add numerical values automatically into sort index+ if(afield.type == field_types::INT32 || afield.type == field_types::INT64 ||+ afield.type == field_types::FLOAT || afield.type == field_types::BOOL ||+ afield.type == field_types::GEOPOINT) {+ spp::sparse_hash_map<uint32_t, int64_t> *doc_to_score = sort_index.at(afield.name);++ bool is_integer = afield.is_integer();+ bool is_float = afield.is_float();+ bool is_bool = afield.is_bool();+ bool is_geopoint = afield.is_geopoint();++ for(const auto& record: iter_batch) {+ if(!record.indexed.ok()) {+ continue;+ }++ const auto& document = record.doc;+ const auto seq_id = record.seq_id;++ if (document.count(afield.name) == 0 || !afield.index) {+ continue;+ }++ if(is_integer) {+ doc_to_score->emplace(seq_id, document[afield.name].get<int64_t>());+ } else if(is_float) {+ int64_t ifloat = float_to_in64_t(document[afield.name].get<float>());+ doc_to_score->emplace(seq_id, ifloat);+ } else if(is_bool) {+ doc_to_score->emplace(seq_id, (int64_t) document[afield.name].get<bool>());+ } else if(is_geopoint) {+ const std::vector<double>& latlong = document[afield.name];+ int64_t lat_lng = GeoPoint::pack_lat_lng(latlong[0], latlong[1]);+ doc_to_score->emplace(seq_id, lat_lng);+ }+ }+ }+ }}void Index::insert_doc(const int64_t score, art_tree *t, uint32_t seq_id,const std::unordered_map<std::string, std::vector<uint32_t>> &token_to_offsets) const {for(auto & kv: token_to_offsets) {- art_document art_doc;- art_doc.id = seq_id;- art_doc.score = score;- art_doc.offsets = kv.second;-+ art_document art_doc(seq_id, score, kv.second);uint32_t num_hits = 0;const unsigned char *key = (const unsigned char *) kv.first.c_str();diff --git a/test/art_test.cpp b/test/art_test.cppindex f0656d53..4682d5f2 100644--- a/test/art_test.cpp+++ b/test/art_test.cpp@@ -14,10 +14,7 @@#define ill_file_path std::string(std::string(ROOT_DIR)+"/test/ill.txt").c_str()art_document get_document(uint32_t id) {- art_document document;- document.score = id;- document.id = id;- document.offsets = {0};+ art_document document(id, id, {0});return document;}@@ -275,24 +272,24 @@ TEST(ArtTest, test_art_iter_prefix) {ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc));s = "api.foo.baz";- doc = get_document((uint32_t) 2);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc));+ auto doc2 = get_document((uint32_t) 2);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc2));s = "api.foe.fum";- doc = get_document((uint32_t) 3);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc));+ auto doc3 = get_document((uint32_t) 3);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc3));s = "abc.123.456";- doc = get_document((uint32_t) 4);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc));+ auto doc4 = get_document((uint32_t) 4);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc4));s = "api.foo";- doc = get_document((uint32_t) 5);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc));+ auto doc5 = get_document((uint32_t) 5);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc5));s = "api";- doc = get_document((uint32_t) 6);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc));+ auto doc6 = get_document((uint32_t) 6);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc6));// Iterate over apiconst char *expected[] = {"api", "api.foe.fum", "api.foo", "api.foo.bar", "api.foo.baz"};@@ -353,13 +350,13 @@ TEST(ArtTest, test_art_long_prefix) {s = "this:key:has:a:long:common:prefix:2";id = 2;- doc = get_document((uint32_t) id);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc));+ auto doc2 = get_document((uint32_t) id);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc2));s = "this:key:has:a:long:common:prefix:1";id = 1;- doc = get_document((uint32_t) id);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc));+ auto doc3 = get_document((uint32_t) id);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc3));// Search for the keyss = "this:key:has:a:long:common:prefix:1";@@ -438,12 +435,12 @@ TEST(ArtTest, test_art_max_prefix_len_scan_prefix) {ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key1, strlen(key1)+1, &doc));const char *key2 = "foobarbaz1-test1-bar";- doc = get_document((uint32_t) 2);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key2, strlen(key2)+1, &doc));+ auto doc2 = get_document((uint32_t) 2);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key2, strlen(key2)+1, &doc2));const char *key3 = "foobarbaz1-test2-foo";- doc = get_document((uint32_t) 3);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key3, strlen(key3)+1, &doc));+ auto doc3 = get_document((uint32_t) 3);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key3, strlen(key3)+1, &doc3));ASSERT_TRUE(art_size(&t) == 3);@@ -469,12 +466,12 @@ TEST(ArtTest, test_art_prefix_iter_out_of_bounds) {ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key1, strlen(key1)+1, &doc));const char *key2 = "foobarbaz1-long-test1-bar";- doc = get_document((uint32_t) 2);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key2, strlen(key2)+1, &doc));+ auto doc2 = get_document((uint32_t) 2);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key2, strlen(key2)+1, &doc2));const char *key3 = "foobarbaz1-long-test2-foo";- doc = get_document((uint32_t) 3);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key3, strlen(key3)+1, &doc));+ auto doc3 = get_document((uint32_t) 3);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key3, strlen(key3)+1, &doc3));ASSERT_TRUE(art_size(&t) == 3);@@ -500,12 +497,12 @@ TEST(ArtTest, test_art_search_out_of_bounds) {ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key1, strlen(key1)+1, &doc));const char *key2 = "foobarbaz1-long-test1-bar";- doc = get_document((uint32_t) 2);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key2, strlen(key2)+1, &doc));+ auto doc2 = get_document((uint32_t) 2);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key2, strlen(key2)+1, &doc2));const char *key3 = "foobarbaz1-long-test2-foo";- doc = get_document((uint32_t) 3);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key3, strlen(key3)+1, &doc));+ auto doc3 = get_document((uint32_t) 3);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key3, strlen(key3)+1, &doc3));ASSERT_TRUE(art_size(&t) == 3);@@ -529,12 +526,12 @@ TEST(ArtTest, test_art_delete_out_of_bounds) {ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key1, strlen(key1)+1, &doc));const char *key2 = "foobarbaz1-long-test1-bar";- doc = get_document((uint32_t) 2);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key2, strlen(key2)+1, &doc));+ art_document doc2 = get_document((uint32_t) 2);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key2, strlen(key2)+1, &doc2));const char *key3 = "foobarbaz1-long-test2-foo";- doc = get_document((uint32_t) 3);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key3, strlen(key3)+1, &doc));+ art_document doc3 = get_document((uint32_t) 3);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key3, strlen(key3)+1, &doc3));ASSERT_TRUE(art_size(&t) == 3);@@ -556,16 +553,16 @@ TEST(ArtTest, test_art_insert_multiple_ids_for_same_token) {art_document doc = get_document((uint32_t) 1);ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key1, strlen(key1)+1, &doc));- doc = get_document((uint32_t) 2);- void* value = art_insert(&t, (unsigned char*)key1, strlen(key1) + 1, &doc);+ art_document doc2 = get_document((uint32_t) 2);+ void* value = art_insert(&t, (unsigned char*)key1, strlen(key1) + 1, &doc2);ASSERT_TRUE(value != NULL);ASSERT_EQ(posting_t::num_ids(value), 2);ASSERT_EQ(posting_t::first_id(value), 1);ASSERT_TRUE(posting_t::contains(value, 2));- doc = get_document((uint32_t) 3);- void* reinsert_value = art_insert(&t, (unsigned char*) key1, strlen(key1) + 1, &doc);+ art_document doc3 = get_document((uint32_t) 3);+ void* reinsert_value = art_insert(&t, (unsigned char*) key1, strlen(key1) + 1, &doc3);ASSERT_TRUE(art_size(&t) == 1);ASSERT_EQ(posting_t::num_ids(reinsert_value), 3);@@ -923,11 +920,11 @@ TEST(ArtTest, test_art_search_ill_like_tokens2) {art_document doc = get_document((uint32_t) 1);ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) keys[0].c_str(), keys[0].size()+1, &doc));- doc = get_document((uint32_t) 2);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) keys[1].c_str(), keys[1].size()+1, &doc));+ art_document doc2 = get_document((uint32_t) 2);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) keys[1].c_str(), keys[1].size()+1, &doc2));- doc = get_document((uint32_t) 3);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) keys[2].c_str(), keys[2].size()+1, &doc));+ art_document doc3 = get_document((uint32_t) 3);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) keys[2].c_str(), keys[2].size()+1, &doc3));for (const auto &key : keys) {art_leaf* l = (art_leaf *) art_search(&t, (const unsigned char *)key.c_str(), key.size()+1);@@ -1099,7 +1096,6 @@ TEST(ArtTest, test_int32_overlap) {art_tree t;art_tree_init(&t);- art_document doc = get_document(1);const int CHAR_LEN = 8;unsigned char chars[CHAR_LEN];@@ -1111,7 +1107,7 @@ TEST(ArtTest, test_int32_overlap) {for(uint32_t i = 0; i < values.size(); i++) {for(size_t j = 0; j < values[i].size(); j++) {encode_int32(values[i][j], chars);- doc.id = i;+ art_document doc = get_document(i);art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc);}}@@ -1174,12 +1170,11 @@ TEST(ArtTest, test_int32_duplicates) {art_tree t;art_tree_init(&t);- art_document doc = get_document(1);const int CHAR_LEN = 8;unsigned char chars[CHAR_LEN];for(size_t i = 0; i < 10000; i++) {- doc.id = i;+ art_document doc = get_document(i);int value = 1900 + (rand() % static_cast<int>(2018 - 1900 + 1));encode_int32(value, chars);art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc);@@ -1499,7 +1494,6 @@ TEST(ArtTest, test_int32_array) {art_tree t;art_tree_init(&t);- art_document doc = get_document(1);const int CHAR_LEN = 8;unsigned char chars[CHAR_LEN];@@ -1514,7 +1508,7 @@ TEST(ArtTest, test_int32_array) {for (uint32_t i = 0; i < values.size(); i++) {for (size_t j = 0; j < values[i].size(); j++) {encode_int32(values[i][j], chars);- doc.id = i;+ art_document doc = get_document(i);art_insert(&t, (unsigned char *) chars, CHAR_LEN, &doc);}}diff --git a/test/collection_test.cpp b/test/collection_test.cppindex fc4a0cd7..52a73c60 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -2261,6 +2261,7 @@ TEST_F(CollectionTest, UpdateDocument) {// reindex the document entirely again verbatim and try queryingadd_op = coll1->add(doc.dump(), UPSERT);ASSERT_TRUE(add_op.ok());+ ASSERT_EQ(1, coll1->get_num_documents());res = coll1->search("lazy", {"title"}, "", {"tags"}, sort_fields, {0}, 10, 1,token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),