Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Field wise indexing.

Kishore Nallan14f4c3e834a2
diff --git a/include/art.h b/include/art.h
index e87117cf..04739e65 100644
--- a/include/art.h
+++ b/include/art.h
@@ -116,11 +116,16 @@ typedef struct {
* Represents a document to be indexed.
* `offsets` refer to the index locations where a token appeared in the document
*/
-typedef struct {
- int64_t score;
- uint32_t id;
- std::vector<uint32_t> offsets;
-} art_document;
+struct art_document {
+ const uint32_t id;
+ const int64_t score;
+ const std::vector<uint32_t> offsets;
+
+ art_document(const uint32_t id, const int64_t score, const std::vector<uint32_t>& offsets):
+ id(id), score(score), offsets(offsets) {
+
+ }
+};
enum token_ordering {
NOT_SET,
@@ -188,6 +193,10 @@ inline uint64_t art_size(art_tree *t) {
*/
void* art_insert(art_tree *t, const unsigned char *key, int key_len, art_document* document);
+/* Insert multiple docs sharing the same key */
+void* art_inserts(art_tree *t, const unsigned char *key, int key_len, const int64_t docs_max_score,
+ std::vector<art_document>& documents);
+
/**
* Deletes a value from the ART tree
* @arg t The tree
diff --git a/include/index.h b/include/index.h
index 7d8f83b8..2b55deb7 100644
--- a/include/index.h
+++ b/include/index.h
@@ -355,6 +355,7 @@ struct index_record {
// pre-processed data primed for indexing
std::unordered_map<std::string, offsets_facet_hashes_t> field_index;
+ int64_t points;
Option<bool> indexed; // indicates if the indexing operation was a success
@@ -681,10 +682,6 @@ public:
Option<uint32_t> remove(const uint32_t seq_id, const nlohmann::json & document, const bool is_update);
- Option<uint32_t> index_in_memory(const index_record& record, uint32_t seq_id,
- const std::string& default_sorting_field,
- const bool is_update);
-
static void validate_and_preprocess(Index *index, std::vector<index_record>& iter_batch,
const size_t batch_start_index, const size_t batch_size,
const std::string & default_sorting_field,
@@ -703,6 +700,11 @@ public:
const std::vector<char>& token_separators,
const std::vector<char>& symbols_to_index);
+ void index_field_in_memory(const field& afield, std::vector<index_record>& iter_batch);
+
+ template<class T>
+ void iterate_and_index_numerical_field(std::vector<index_record>& iter_batch, const field& afield, T func);
+
//static bool is_point_in_polygon(const Geofence& poly, const GeoCoord& point);
//static double transform_for_180th_meridian(Geofence& poly);
@@ -753,3 +755,20 @@ public:
std::vector<facet_info_t>& facet_infos) const;
};
+template<class T>
+void Index::iterate_and_index_numerical_field(std::vector<index_record>& iter_batch, const field& afield, T func) {
+ for(const auto& record: iter_batch) {
+ if(!record.indexed.ok()) {
+ continue;
+ }
+
+ const auto& document = record.doc;
+ const auto seq_id = record.seq_id;
+
+ if (document.count(afield.name) == 0 || !afield.index) {
+ continue;
+ }
+
+ func(record, seq_id);
+ }
+}
diff --git a/src/art.cpp b/src/art.cpp
index cba9cb04..40017d0c 100644
--- a/src/art.cpp
+++ b/src/art.cpp
@@ -421,7 +421,6 @@ static void add_document_to_leaf(art_document *document, art_leaf *leaf) {
if(document->score == USE_FREQUENCY_SCORE) {
leaf->max_score = posting_t::num_ids(leaf->values);
- document->score = leaf->max_score;
}
}
@@ -609,11 +608,17 @@ static int prefix_mismatch(const art_node *n, const unsigned char *key, int key_
return idx;
}
-static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *key, uint32_t key_len,
- art_document *document, int depth, std::list<art_node*>& path, int *old) {
+static void* recursive_insert(art_node* n, art_node** ref, const unsigned char* key, uint32_t key_len,
+ const int64_t docs_max_score, std::vector<art_document>& documents, int depth,
+ std::list<art_node*>& path, int* old) {
// If we are at a NULL node, inject a leaf
if (!n) {
- *ref = (art_node*)SET_LEAF(make_leaf(key, key_len, document));
+ art_leaf* new_leaf = make_leaf(key, key_len, &documents[0]);
+ for(size_t i = 1; i < documents.size(); i++) {
+ add_document_to_leaf(&documents[i], new_leaf);
+ }
+
+ *ref = (art_node*)SET_LEAF(new_leaf);
return NULL;
}
@@ -624,7 +629,9 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *
// Check if we are updating an existing value
if (!leaf_matches(l, key, key_len, depth)) {
*old = 1;
- add_document_to_leaf(document, l);
+ for(size_t i = 0; i < documents.size(); i++) {
+ add_document_to_leaf(&documents[i], l);
+ }
return l->values;
}
@@ -632,12 +639,16 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *
art_node4 *new_n = (art_node4*)alloc_node(NODE4);
// Create a new leaf
- art_leaf *l2 = make_leaf(key, key_len, document);
+ art_leaf *l2 = make_leaf(key, key_len, &documents[0]);
uint32_t longest_prefix = longest_common_prefix(l, l2, depth);
new_n->n.partial_len = longest_prefix;
memcpy(new_n->n.partial, key+depth, min(MAX_PREFIX_LEN, longest_prefix));
+ for(size_t i = 1; i < documents.size(); i++) {
+ add_document_to_leaf(&documents[i], l2);
+ }
+
// Add the leafs to the new node4
*ref = (art_node*)new_n;
add_child4(new_n, ref, l->key[depth+longest_prefix], SET_LEAF(l));
@@ -645,8 +656,8 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *
return NULL;
}
- if(document->score != USE_FREQUENCY_SCORE) {
- n->max_score = MAX(n->max_score, document->score);
+ if(docs_max_score != USE_FREQUENCY_SCORE) {
+ n->max_score = MAX(n->max_score, docs_max_score);
}
// Check if given node has a prefix
@@ -679,7 +690,11 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *
}
// Insert the new leaf
- art_leaf *l = make_leaf(key, key_len, document);
+ art_leaf *l = make_leaf(key, key_len, &documents[0]);
+ for(size_t i = 1; i < documents.size(); i++) {
+ add_document_to_leaf(&documents[i], l);
+ }
+
add_child4(new_n, ref, key[depth+prefix_diff], SET_LEAF(l));
path.push_back(*ref);
return NULL;
@@ -690,11 +705,15 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *
// Find a child to recurse to
art_node **child = find_child(n, key[depth]);
if (child) {
- return recursive_insert(*child, child, key, key_len, document, depth + 1, path, old);
+ return recursive_insert(*child, child, key, key_len, docs_max_score, documents, depth + 1, path, old);
}
// No child, node goes within us
- art_leaf *l = make_leaf(key, key_len, document);
+ art_leaf *l = make_leaf(key, key_len, &documents[0]);
+ for(size_t i = 1; i < documents.size(); i++) {
+ add_document_to_leaf(&documents[i], l);
+ }
+
add_child(n, ref, key[depth], SET_LEAF(l));
path.push_back(*ref);
return NULL;
@@ -710,16 +729,22 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *
* the old value pointer is returned.
*/
void* art_insert(art_tree *t, const unsigned char *key, int key_len, art_document* document) {
+ std::vector<art_document> documents = {*document};
+ return art_inserts(t, key, key_len, document->score, documents);
+}
+
+void* art_inserts(art_tree *t, const unsigned char *key, int key_len, const int64_t docs_max_score,
+ std::vector<art_document>& documents) {
int old_val = 0;
std::list<art_node*> path;
- bool frequency_based_ordering = (document->score == USE_FREQUENCY_SCORE);
- void *old = recursive_insert(t->root, &t->root, key, key_len, document, 0, path, &old_val);
+ bool frequency_based_ordering = (docs_max_score == USE_FREQUENCY_SCORE);
+ void *old = recursive_insert(t->root, &t->root, key, key_len, docs_max_score, documents, 0, path, &old_val);
if (!old_val) t->size++;
if(frequency_based_ordering) {
for(art_node* n: path) {
- n->max_score = MAX(n->max_score, document->score);
+ n->max_score = MAX(n->max_score, docs_max_score);
}
}
diff --git a/src/collection.cpp b/src/collection.cpp
index a28775d3..1dd85f3a 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -368,7 +368,11 @@ Option<uint32_t> Collection::index_in_memory(nlohmann::json &document, uint32_t
index_record rec(0, seq_id, document, op, dirty_values);
Index::compute_token_offsets_facets(rec, search_schema, facet_schema, token_separators, symbols_to_index);
- index->index_in_memory(rec, seq_id, default_sorting_field, op);
+
+ std::vector<index_record> index_batch;
+ index_batch.emplace_back(std::move(rec));
+ Index::batch_memory_index(index, index_batch, default_sorting_field, search_schema, facet_schema,
+ fallback_field_type, token_separators, symbols_to_index);
num_documents += 1;
return Option<>(200);
@@ -2429,6 +2433,10 @@ Index* Collection::init_index() {
continue;
}
+ if(field.name == ".*") {
+ continue;
+ }
+
search_schema.emplace(field.name, field);
if(field.is_facet()) {
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 7dae91d3..e3307c05 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -161,7 +161,13 @@ Option<bool> CollectionManager::load(const size_t collection_batch_size, const s
auto captured_store = store;
loading_pool.enqueue([captured_store, num_collections, collection_meta, document_batch_size,
&m_process, &cv_process, &num_processed, &next_coll_id_status, quit = quit]() {
+
+ //auto begin = std::chrono::high_resolution_clock::now();
Option<bool> res = load_collection(collection_meta, document_batch_size, next_coll_id_status, *quit);
+ /*long long int timeMillis =
+ std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();
+ LOG(INFO) << "Time taken for indexing: " << timeMillis << "ms";*/
+
if(!res.ok()) {
LOG(ERROR) << "Error while loading collection. " << res.error();
LOG(ERROR) << "Typesense is quitting.";
diff --git a/src/index.cpp b/src/index.cpp
index 22341a8f..403f020c 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -18,6 +18,7 @@
#include <s2/s2builder.h>
#include <posting.h>
#include <thread_local_vars.h>
+#include <unordered_set>
#include "logger.h"
#define RETURN_CIRCUIT_BREAKER if(std::chrono::duration_cast<std::chrono::milliseconds>(\
@@ -252,182 +253,6 @@ void Index::compute_token_offsets_facets(index_record& record,
}
}
-Option<uint32_t> Index::index_in_memory(const index_record& record, uint32_t seq_id,
- const std::string & default_sorting_field,
- const bool is_update) {
- std::unique_lock lock(mutex);
-
- int64_t points = 0;
- auto& document = record.doc;
-
- if(document.count(default_sorting_field) == 0) {
- auto default_sorting_field_it = sort_index.find(default_sorting_field);
- if(default_sorting_field_it != sort_index.end()) {
- auto seq_id_it = default_sorting_field_it->second->find(seq_id);
- if(seq_id_it != default_sorting_field_it->second->end()) {
- points = seq_id_it->second;
- } else {
- points = INT64_MIN;
- }
- } else {
- points = INT64_MIN;
- }
- } else {
- points = get_points_from_doc(document, default_sorting_field);
- }
-
- if(!is_update) {
- // for updates, the seq_id will already exist
- seq_ids.append(seq_id);
- }
-
- // assumes that validation has already been done
- for(const auto& field_pair: search_schema) {
- const std::string & field_name = field_pair.first;
-
- if(document.count(field_name) == 0 || !field_pair.second.index) {
- continue;
- }
-
- bool is_facet = (facet_schema.count(field_name) != 0);
-
- // non-string, non-geo faceted field should be indexed as faceted string field as well
- if(field_pair.second.facet && !field_pair.second.is_string() && !field_pair.second.is_geopoint()) {
- auto field_index_it = record.field_index.find(field_name);
- if(field_index_it == record.field_index.end()) {
- continue;
- }
-
- art_tree *t = search_index.at(field_pair.second.faceted_name());
-
- index_strings_field(points, t, seq_id, is_facet, field_pair.second,
- field_index_it->second.offsets,
- field_index_it->second.facet_hashes);
- }
-
- if(field_pair.second.is_string()) {
- auto field_index_it = record.field_index.find(field_name);
- if(field_index_it == record.field_index.end()) {
- continue;
- }
-
- art_tree *t = search_index.at(field_name);
- index_strings_field(points, t, seq_id, is_facet, field_pair.second,
- field_index_it->second.offsets,
- field_index_it->second.facet_hashes);
- } else if(field_pair.second.type == field_types::INT32) {
- auto num_tree = numerical_index.at(field_name);
- int32_t value = document[field_name].get<int32_t>();
- num_tree->insert(value, seq_id);
- } else if(field_pair.second.type == field_types::INT64) {
- auto num_tree = numerical_index.at(field_name);
- int64_t value = document[field_name].get<int64_t>();
- num_tree->insert(value, seq_id);
- } else if(field_pair.second.type == field_types::FLOAT) {
- auto num_tree = numerical_index.at(field_name);
- float fvalue = document[field_name].get<float>();
- int64_t value = float_to_in64_t(fvalue);
- num_tree->insert(value, seq_id);
- } else if(field_pair.second.type == field_types::BOOL) {
- auto num_tree = numerical_index.at(field_name);
- bool value = document[field_name];
- num_tree->insert(value, seq_id);
- } else if(field_pair.second.type == field_types::GEOPOINT) {
- const std::vector<double>& latlong = document[field_name];
- auto geo_index = geopoint_index.at(field_name);
-
- S2RegionTermIndexer::Options options;
- options.set_index_contains_points_only(true);
- S2RegionTermIndexer indexer(options);
- S2Point point = S2LatLng::FromDegrees(latlong[0], latlong[1]).ToPoint();
-
- for(const auto& term: indexer.GetIndexTerms(point, "")) {
- (*geo_index)[term].push_back(seq_id);
- }
- } else if(field_pair.second.type == field_types::GEOPOINT_ARRAY) {
- const std::vector<std::vector<double>>& latlongs = document[field_name];
- auto geo_index = geopoint_index.at(field_name);
- S2RegionTermIndexer::Options options;
- options.set_index_contains_points_only(true);
- S2RegionTermIndexer indexer(options);
-
- int64_t* packed_latlongs = new int64_t[latlongs.size() + 1];
- packed_latlongs[0] = latlongs.size();
-
- for(size_t li = 0; li < latlongs.size(); li++) {
- auto& latlong = latlongs[li];
- S2Point point = S2LatLng::FromDegrees(latlong[0], latlong[1]).ToPoint();
- std::set<std::string> terms;
- for(const auto& term: indexer.GetIndexTerms(point, "")) {
- terms.insert(term);
- }
-
- for(const auto& term: terms) {
- (*geo_index)[term].push_back(seq_id);
- }
-
- int64_t packed_latlong = GeoPoint::pack_lat_lng(latlong[0], latlong[1]);
- packed_latlongs[li + 1] = packed_latlong;
- }
-
- geo_array_index.at(field_name)->emplace(seq_id, packed_latlongs);
- }
-
- else if(field_pair.second.is_array()) {
- // all other numerical arrays
- auto num_tree = numerical_index.at(field_name);
-
- for(size_t arr_i = 0; arr_i < document[field_name].size(); arr_i++) {
- const auto& arr_value = document[field_name][arr_i];
-
- if(field_pair.second.type == field_types::INT32_ARRAY) {
- const int32_t value = arr_value;
- num_tree->insert(value, seq_id);
- }
-
- else if(field_pair.second.type == field_types::INT64_ARRAY) {
- const int64_t value = arr_value;
- num_tree->insert(value, seq_id);
- }
-
- else if(field_pair.second.type == field_types::FLOAT_ARRAY) {
- const float fvalue = arr_value;
- int64_t value = float_to_in64_t(fvalue);
- num_tree->insert(value, seq_id);
- }
-
- else if(field_pair.second.type == field_types::BOOL_ARRAY) {
- const bool value = document[field_name][arr_i];
- num_tree->insert(int64_t(value), seq_id);
- }
- }
- }
-
- // add numerical values automatically into sort index
- if(field_pair.second.type == field_types::INT32 || field_pair.second.type == field_types::INT64 ||
- field_pair.second.type == field_types::FLOAT || field_pair.second.type == field_types::BOOL ||
- field_pair.second.type == field_types::GEOPOINT) {
- spp::sparse_hash_map<uint32_t, int64_t> *doc_to_score = sort_index.at(field_pair.first);
-
- if(field_pair.second.is_integer() ) {
- doc_to_score->emplace(seq_id, document[field_pair.first].get<int64_t>());
- } else if(field_pair.second.is_float()) {
- int64_t ifloat = float_to_in64_t(document[field_pair.first].get<float>());
- doc_to_score->emplace(seq_id, ifloat);
- } else if(field_pair.second.is_bool()) {
- doc_to_score->emplace(seq_id, (int64_t) document[field_pair.first].get<bool>());
- } else if(field_pair.second.is_geopoint()) {
- const std::vector<double>& latlong = document[field_pair.first];
- int64_t lat_lng = GeoPoint::pack_lat_lng(latlong[0], latlong[1]);
- doc_to_score->emplace(seq_id, lat_lng);
- }
- }
- }
-
- num_documents += 1;
- return Option<>(201);
-}
-
Option<uint32_t> Index::validate_index_in_memory(nlohmann::json& document, uint32_t seq_id,
const std::string & default_sorting_field,
const std::unordered_map<std::string, field> & search_schema,
@@ -628,6 +453,27 @@ void Index::validate_and_preprocess(Index *index, std::vector<index_record>& ite
}
compute_token_offsets_facets(index_rec, search_schema, facet_schema, token_separators, symbols_to_index);
+
+ int64_t points = 0;
+
+ if(index_rec.doc.count(default_sorting_field) == 0) {
+ auto default_sorting_field_it = index->sort_index.find(default_sorting_field);
+ if(default_sorting_field_it != index->sort_index.end()) {
+ auto seq_id_it = default_sorting_field_it->second->find(index_rec.seq_id);
+ if(seq_id_it != default_sorting_field_it->second->end()) {
+ points = seq_id_it->second;
+ } else {
+ points = INT64_MIN;
+ }
+ } else {
+ points = INT64_MIN;
+ }
+ } else {
+ points = get_points_from_doc(index_rec.doc, default_sorting_field);
+ }
+
+ index_rec.points = points;
+ index_rec.index_success();
}
}
@@ -674,51 +520,290 @@ size_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_b
batch_index += batch_len;
}
- std::unique_lock<std::mutex> lock_process(m_process);
- cv_process.wait(lock_process, [&](){ return num_processed == num_queued; });
+ {
+ std::unique_lock<std::mutex> lock_process(m_process);
+ cv_process.wait(lock_process, [&](){ return num_processed == num_queued; });
+ }
- Option<uint32_t> index_mem_op(0);
+ std::unordered_set<std::string> found_fields;
- for(auto& index_rec: iter_batch) {
- if(index_rec.operation == DELETE) {
- continue;
+ for(size_t i = 0; i < iter_batch.size(); i++) {
+ auto& index_rec = iter_batch[i];
+ if(index_rec.is_update) {
+ index->remove(index_rec.seq_id, index_rec.del_doc, index_rec.is_update);
+ } else {
+ num_indexed++;
}
- if(!index_rec.indexed.ok()) {
- // some records could have been invalidated upstream
+ for(const auto& kv: index_rec.doc.items()) {
+ found_fields.insert(kv.key());
+ }
+ }
+
+ num_queued = num_processed = 0;
+
+ for(const auto& field_name: found_fields) {
+ //LOG(INFO) << "field name: " << field_name;
+ if(field_name != "id" && search_schema.count(field_name) == 0) {
continue;
}
- if(index_rec.is_update) {
- index->remove(index_rec.seq_id, index_rec.del_doc, index_rec.is_update);
+ num_queued++;
+
+ index->thread_pool->enqueue([&]() {
+ const field& f = (field_name == "id") ?
+ field("id", field_types::STRING, false) : search_schema.at(field_name);
+ index->index_field_in_memory(f, iter_batch);
+ std::unique_lock<std::mutex> lock(m_process);
+ num_processed++;
+ cv_process.notify_one();
+ });
+ }
+
+ {
+ std::unique_lock<std::mutex> lock_process(m_process);
+ cv_process.wait(lock_process, [&](){ return num_processed == num_queued; });
+ }
+
+ return num_indexed;
+}
+
+void Index::index_field_in_memory(const field& afield, std::vector<index_record>& iter_batch) {
+ // indexes a given field of all documents in the batch
+
+ if(afield.name == "id") {
+ for(const auto& record: iter_batch) {
+ if(!record.is_update) {
+ // for updates, the seq_id will already exist
+ seq_ids.append(record.seq_id);
+ }
}
- try {
- index_mem_op = index->index_in_memory(index_rec, index_rec.seq_id, default_sorting_field, index_rec.is_update);
- } catch(const std::exception& e) {
- const std::string& error_msg = std::string("Fatal error during indexing: ") + e.what();
- LOG(ERROR) << error_msg << ", document: " << index_rec.doc;
- index_mem_op = Option<uint32_t>(500, error_msg);
+ return;
+ }
+
+ // We have to handle both these edge cases:
+ // a) `afield` might not exist in the document (optional field)
+ // b) `afield` value could be empty
+
+ // non-geo faceted field should be indexed as faceted string field as well
+ bool non_string_facet_field = (afield.facet && !afield.is_geopoint());
+
+ if(afield.is_string() || non_string_facet_field) {
+ std::unordered_map<std::string, std::vector<art_document>> token_to_doc_offsets;
+ int64_t max_score = INT64_MIN;
+
+ for(const auto& record: iter_batch) {
+ const auto& document = record.doc;
+ const auto seq_id = record.seq_id;
+
+ if(document.count(afield.name) == 0 || !afield.index) {
+ continue;
+ }
+
+ auto field_index_it = record.field_index.find(afield.name);
+ if(field_index_it == record.field_index.end()) {
+ continue;
+ }
+
+ if(afield.facet) {
+ facet_hash_values_t fhashvalues;
+ fhashvalues.length = field_index_it->second.facet_hashes.size();
+ fhashvalues.hashes = new uint64_t[field_index_it->second.facet_hashes.size()];
+
+ for(size_t i = 0; i < field_index_it->second.facet_hashes.size(); i++) {
+ fhashvalues.hashes[i] = field_index_it->second.facet_hashes[i];
+ }
+
+ facet_index_v3[afield.name][seq_id % ARRAY_FACET_DIM]->emplace(seq_id, std::move(fhashvalues));
+ }
+
+ if(record.points > max_score) {
+ max_score = record.points;
+ }
+
+ for(auto &token_offsets: field_index_it->second.offsets) {
+ token_to_doc_offsets[token_offsets.first].emplace_back(seq_id, record.points, token_offsets.second);
+ }
}
- index_rec.index_success();
+ auto tree_it = search_index.find(afield.faceted_name());
+ if(tree_it == search_index.end()) {
+ return;
+ }
- if(!index_rec.is_update) {
- num_indexed++;
+ art_tree *t = tree_it->second;
+
+ for(auto& token_to_doc: token_to_doc_offsets) {
+ std::vector<art_document>& documents = token_to_doc.second;
+ const std::string& token = token_to_doc.first;
+
+ const auto *key = (const unsigned char *) token.c_str();
+ int key_len = (int) token.length() + 1; // for the terminating \0 char
+
+ //LOG(INFO) << "key: " << key << ", art_doc.id: " << art_doc.id;
+ art_inserts(t, key, key_len, max_score, documents);
}
}
- return num_indexed;
+ if(!afield.is_string()) {
+ if (afield.type == field_types::INT32) {
+ auto num_tree = numerical_index.at(afield.name);
+ iterate_and_index_numerical_field(iter_batch, afield, [&afield, num_tree]
+ (const index_record& record, uint32_t seq_id) {
+ int32_t value = record.doc[afield.name].get<int32_t>();
+ num_tree->insert(value, seq_id);
+ });
+ }
+
+ else if(afield.type == field_types::INT64) {
+ auto num_tree = numerical_index.at(afield.name);
+ iterate_and_index_numerical_field(iter_batch, afield, [&afield, num_tree]
+ (const index_record& record, uint32_t seq_id) {
+ int64_t value = record.doc[afield.name].get<int64_t>();
+ num_tree->insert(value, seq_id);
+ });
+ }
+
+ else if(afield.type == field_types::FLOAT) {
+ auto num_tree = numerical_index.at(afield.name);
+ iterate_and_index_numerical_field(iter_batch, afield, [&afield, num_tree]
+ (const index_record& record, uint32_t seq_id) {
+ float fvalue = record.doc[afield.name].get<float>();
+ int64_t value = float_to_in64_t(fvalue);
+ num_tree->insert(value, seq_id);
+ });
+ } else if(afield.type == field_types::BOOL) {
+ auto num_tree = numerical_index.at(afield.name);
+ iterate_and_index_numerical_field(iter_batch, afield, [&afield, num_tree]
+ (const index_record& record, uint32_t seq_id) {
+ bool value = record.doc[afield.name].get<bool>();
+ num_tree->insert(value, seq_id);
+ });
+ } else if(afield.type == field_types::GEOPOINT) {
+ auto geo_index = geopoint_index.at(afield.name);
+
+ iterate_and_index_numerical_field(iter_batch, afield, [&afield, geo_index]
+ (const index_record& record, uint32_t seq_id) {
+ const std::vector<double>& latlong = record.doc[afield.name];
+
+ S2RegionTermIndexer::Options options;
+ options.set_index_contains_points_only(true);
+ S2RegionTermIndexer indexer(options);
+ S2Point point = S2LatLng::FromDegrees(latlong[0], latlong[1]).ToPoint();
+
+ for(const auto& term: indexer.GetIndexTerms(point, "")) {
+ (*geo_index)[term].push_back(seq_id);
+ }
+ });
+ } else if(afield.type == field_types::GEOPOINT_ARRAY) {
+ auto geo_index = geopoint_index.at(afield.name);
+
+ iterate_and_index_numerical_field(iter_batch, afield,
+ [&afield, &geo_array_index=geo_array_index, geo_index](const index_record& record, uint32_t seq_id) {
+
+ const std::vector<std::vector<double>>& latlongs = record.doc[afield.name];
+ S2RegionTermIndexer::Options options;
+ options.set_index_contains_points_only(true);
+ S2RegionTermIndexer indexer(options);
+
+ int64_t* packed_latlongs = new int64_t[latlongs.size() + 1];
+ packed_latlongs[0] = latlongs.size();
+
+ for(size_t li = 0; li < latlongs.size(); li++) {
+ auto& latlong = latlongs[li];
+ S2Point point = S2LatLng::FromDegrees(latlong[0], latlong[1]).ToPoint();
+ std::set<std::string> terms;
+ for(const auto& term: indexer.GetIndexTerms(point, "")) {
+ terms.insert(term);
+ }
+
+ for(const auto& term: terms) {
+ (*geo_index)[term].push_back(seq_id);
+ }
+
+ int64_t packed_latlong = GeoPoint::pack_lat_lng(latlong[0], latlong[1]);
+ packed_latlongs[li + 1] = packed_latlong;
+ }
+
+ geo_array_index.at(afield.name)->emplace(seq_id, packed_latlongs);
+ });
+ } else if(afield.is_array()) {
+ // all other numerical arrays
+ auto num_tree = numerical_index.at(afield.name);
+ iterate_and_index_numerical_field(iter_batch, afield, [&afield, num_tree]
+ (const index_record& record, uint32_t seq_id) {
+ for(size_t arr_i = 0; arr_i < record.doc[afield.name].size(); arr_i++) {
+ const auto& arr_value = record.doc[afield.name][arr_i];
+
+ if(afield.type == field_types::INT32_ARRAY) {
+ const int32_t value = arr_value;
+ num_tree->insert(value, seq_id);
+ }
+
+ else if(afield.type == field_types::INT64_ARRAY) {
+ const int64_t value = arr_value;
+ num_tree->insert(value, seq_id);
+ }
+
+ else if(afield.type == field_types::FLOAT_ARRAY) {
+ const float fvalue = arr_value;
+ int64_t value = float_to_in64_t(fvalue);
+ num_tree->insert(value, seq_id);
+ }
+
+ else if(afield.type == field_types::BOOL_ARRAY) {
+ const bool value = record.doc[afield.name][arr_i];
+ num_tree->insert(int64_t(value), seq_id);
+ }
+ }
+ });
+ }
+
+ // add numerical values automatically into sort index
+ if(afield.type == field_types::INT32 || afield.type == field_types::INT64 ||
+ afield.type == field_types::FLOAT || afield.type == field_types::BOOL ||
+ afield.type == field_types::GEOPOINT) {
+ spp::sparse_hash_map<uint32_t, int64_t> *doc_to_score = sort_index.at(afield.name);
+
+ bool is_integer = afield.is_integer();
+ bool is_float = afield.is_float();
+ bool is_bool = afield.is_bool();
+ bool is_geopoint = afield.is_geopoint();
+
+ for(const auto& record: iter_batch) {
+ if(!record.indexed.ok()) {
+ continue;
+ }
+
+ const auto& document = record.doc;
+ const auto seq_id = record.seq_id;
+
+ if (document.count(afield.name) == 0 || !afield.index) {
+ continue;
+ }
+
+ if(is_integer) {
+ doc_to_score->emplace(seq_id, document[afield.name].get<int64_t>());
+ } else if(is_float) {
+ int64_t ifloat = float_to_in64_t(document[afield.name].get<float>());
+ doc_to_score->emplace(seq_id, ifloat);
+ } else if(is_bool) {
+ doc_to_score->emplace(seq_id, (int64_t) document[afield.name].get<bool>());
+ } else if(is_geopoint) {
+ const std::vector<double>& latlong = document[afield.name];
+ int64_t lat_lng = GeoPoint::pack_lat_lng(latlong[0], latlong[1]);
+ doc_to_score->emplace(seq_id, lat_lng);
+ }
+ }
+ }
+ }
}
void Index::insert_doc(const int64_t score, art_tree *t, uint32_t seq_id,
const std::unordered_map<std::string, std::vector<uint32_t>> &token_to_offsets) const {
for(auto & kv: token_to_offsets) {
- art_document art_doc;
- art_doc.id = seq_id;
- art_doc.score = score;
- art_doc.offsets = kv.second;
-
+ art_document art_doc(seq_id, score, kv.second);
uint32_t num_hits = 0;
const unsigned char *key = (const unsigned char *) kv.first.c_str();
diff --git a/test/art_test.cpp b/test/art_test.cpp
index f0656d53..4682d5f2 100644
--- a/test/art_test.cpp
+++ b/test/art_test.cpp
@@ -14,10 +14,7 @@
#define ill_file_path std::string(std::string(ROOT_DIR)+"/test/ill.txt").c_str()
art_document get_document(uint32_t id) {
- art_document document;
- document.score = id;
- document.id = id;
- document.offsets = {0};
+ art_document document(id, id, {0});
return document;
}
@@ -275,24 +272,24 @@ TEST(ArtTest, test_art_iter_prefix) {
ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc));
s = "api.foo.baz";
- doc = get_document((uint32_t) 2);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc));
+ auto doc2 = get_document((uint32_t) 2);
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc2));
s = "api.foe.fum";
- doc = get_document((uint32_t) 3);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc));
+ auto doc3 = get_document((uint32_t) 3);
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc3));
s = "abc.123.456";
- doc = get_document((uint32_t) 4);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc));
+ auto doc4 = get_document((uint32_t) 4);
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc4));
s = "api.foo";
- doc = get_document((uint32_t) 5);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc));
+ auto doc5 = get_document((uint32_t) 5);
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc5));
s = "api";
- doc = get_document((uint32_t) 6);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc));
+ auto doc6 = get_document((uint32_t) 6);
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc6));
// Iterate over api
const char *expected[] = {"api", "api.foe.fum", "api.foo", "api.foo.bar", "api.foo.baz"};
@@ -353,13 +350,13 @@ TEST(ArtTest, test_art_long_prefix) {
s = "this:key:has:a:long:common:prefix:2";
id = 2;
- doc = get_document((uint32_t) id);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc));
+ auto doc2 = get_document((uint32_t) id);
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc2));
s = "this:key:has:a:long:common:prefix:1";
id = 1;
- doc = get_document((uint32_t) id);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc));
+ auto doc3 = get_document((uint32_t) id);
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc3));
// Search for the keys
s = "this:key:has:a:long:common:prefix:1";
@@ -438,12 +435,12 @@ TEST(ArtTest, test_art_max_prefix_len_scan_prefix) {
ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key1, strlen(key1)+1, &doc));
const char *key2 = "foobarbaz1-test1-bar";
- doc = get_document((uint32_t) 2);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key2, strlen(key2)+1, &doc));
+ auto doc2 = get_document((uint32_t) 2);
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key2, strlen(key2)+1, &doc2));
const char *key3 = "foobarbaz1-test2-foo";
- doc = get_document((uint32_t) 3);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key3, strlen(key3)+1, &doc));
+ auto doc3 = get_document((uint32_t) 3);
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key3, strlen(key3)+1, &doc3));
ASSERT_TRUE(art_size(&t) == 3);
@@ -469,12 +466,12 @@ TEST(ArtTest, test_art_prefix_iter_out_of_bounds) {
ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key1, strlen(key1)+1, &doc));
const char *key2 = "foobarbaz1-long-test1-bar";
- doc = get_document((uint32_t) 2);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key2, strlen(key2)+1, &doc));
+ auto doc2 = get_document((uint32_t) 2);
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key2, strlen(key2)+1, &doc2));
const char *key3 = "foobarbaz1-long-test2-foo";
- doc = get_document((uint32_t) 3);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key3, strlen(key3)+1, &doc));
+ auto doc3 = get_document((uint32_t) 3);
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key3, strlen(key3)+1, &doc3));
ASSERT_TRUE(art_size(&t) == 3);
@@ -500,12 +497,12 @@ TEST(ArtTest, test_art_search_out_of_bounds) {
ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key1, strlen(key1)+1, &doc));
const char *key2 = "foobarbaz1-long-test1-bar";
- doc = get_document((uint32_t) 2);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key2, strlen(key2)+1, &doc));
+ auto doc2 = get_document((uint32_t) 2);
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key2, strlen(key2)+1, &doc2));
const char *key3 = "foobarbaz1-long-test2-foo";
- doc = get_document((uint32_t) 3);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key3, strlen(key3)+1, &doc));
+ auto doc3 = get_document((uint32_t) 3);
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key3, strlen(key3)+1, &doc3));
ASSERT_TRUE(art_size(&t) == 3);
@@ -529,12 +526,12 @@ TEST(ArtTest, test_art_delete_out_of_bounds) {
ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key1, strlen(key1)+1, &doc));
const char *key2 = "foobarbaz1-long-test1-bar";
- doc = get_document((uint32_t) 2);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key2, strlen(key2)+1, &doc));
+ art_document doc2 = get_document((uint32_t) 2);
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key2, strlen(key2)+1, &doc2));
const char *key3 = "foobarbaz1-long-test2-foo";
- doc = get_document((uint32_t) 3);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key3, strlen(key3)+1, &doc));
+ art_document doc3 = get_document((uint32_t) 3);
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key3, strlen(key3)+1, &doc3));
ASSERT_TRUE(art_size(&t) == 3);
@@ -556,16 +553,16 @@ TEST(ArtTest, test_art_insert_multiple_ids_for_same_token) {
art_document doc = get_document((uint32_t) 1);
ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key1, strlen(key1)+1, &doc));
- doc = get_document((uint32_t) 2);
- void* value = art_insert(&t, (unsigned char*)key1, strlen(key1) + 1, &doc);
+ art_document doc2 = get_document((uint32_t) 2);
+ void* value = art_insert(&t, (unsigned char*)key1, strlen(key1) + 1, &doc2);
ASSERT_TRUE(value != NULL);
ASSERT_EQ(posting_t::num_ids(value), 2);
ASSERT_EQ(posting_t::first_id(value), 1);
ASSERT_TRUE(posting_t::contains(value, 2));
- doc = get_document((uint32_t) 3);
- void* reinsert_value = art_insert(&t, (unsigned char*) key1, strlen(key1) + 1, &doc);
+ art_document doc3 = get_document((uint32_t) 3);
+ void* reinsert_value = art_insert(&t, (unsigned char*) key1, strlen(key1) + 1, &doc3);
ASSERT_TRUE(art_size(&t) == 1);
ASSERT_EQ(posting_t::num_ids(reinsert_value), 3);
@@ -923,11 +920,11 @@ TEST(ArtTest, test_art_search_ill_like_tokens2) {
art_document doc = get_document((uint32_t) 1);
ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) keys[0].c_str(), keys[0].size()+1, &doc));
- doc = get_document((uint32_t) 2);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) keys[1].c_str(), keys[1].size()+1, &doc));
+ art_document doc2 = get_document((uint32_t) 2);
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) keys[1].c_str(), keys[1].size()+1, &doc2));
- doc = get_document((uint32_t) 3);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) keys[2].c_str(), keys[2].size()+1, &doc));
+ art_document doc3 = get_document((uint32_t) 3);
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) keys[2].c_str(), keys[2].size()+1, &doc3));
for (const auto &key : keys) {
art_leaf* l = (art_leaf *) art_search(&t, (const unsigned char *)key.c_str(), key.size()+1);
@@ -1099,7 +1096,6 @@ TEST(ArtTest, test_int32_overlap) {
art_tree t;
art_tree_init(&t);
- art_document doc = get_document(1);
const int CHAR_LEN = 8;
unsigned char chars[CHAR_LEN];
@@ -1111,7 +1107,7 @@ TEST(ArtTest, test_int32_overlap) {
for(uint32_t i = 0; i < values.size(); i++) {
for(size_t j = 0; j < values[i].size(); j++) {
encode_int32(values[i][j], chars);
- doc.id = i;
+ art_document doc = get_document(i);
art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc);
}
}
@@ -1174,12 +1170,11 @@ TEST(ArtTest, test_int32_duplicates) {
art_tree t;
art_tree_init(&t);
- art_document doc = get_document(1);
const int CHAR_LEN = 8;
unsigned char chars[CHAR_LEN];
for(size_t i = 0; i < 10000; i++) {
- doc.id = i;
+ art_document doc = get_document(i);
int value = 1900 + (rand() % static_cast<int>(2018 - 1900 + 1));
encode_int32(value, chars);
art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc);
@@ -1499,7 +1494,6 @@ TEST(ArtTest, test_int32_array) {
art_tree t;
art_tree_init(&t);
- art_document doc = get_document(1);
const int CHAR_LEN = 8;
unsigned char chars[CHAR_LEN];
@@ -1514,7 +1508,7 @@ TEST(ArtTest, test_int32_array) {
for (uint32_t i = 0; i < values.size(); i++) {
for (size_t j = 0; j < values[i].size(); j++) {
encode_int32(values[i][j], chars);
- doc.id = i;
+ art_document doc = get_document(i);
art_insert(&t, (unsigned char *) chars, CHAR_LEN, &doc);
}
}
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index fc4a0cd7..52a73c60 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -2261,6 +2261,7 @@ TEST_F(CollectionTest, UpdateDocument) {
// reindex the document entirely again verbatim and try querying
add_op = coll1->add(doc.dump(), UPSERT);
ASSERT_TRUE(add_op.ok());
+ ASSERT_EQ(1, coll1->get_num_documents());
res = coll1->search("lazy", {"title"}, "", {"tags"}, sort_fields, {0}, 10, 1,
token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),