Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Posting list integration.
Kishore Nallan929b6796ccf4
diff --git a/TODO.md b/TODO.mdindex fdbf335c..0a380d7a 100644--- a/TODO.md+++ b/TODO.md@@ -3,6 +3,9 @@- Writes on dedicated threads- Single field aggregation short circuit- Test for group + multiple fields+- Intersect with single posting list+- Test for erase dropping elements below compressed list threshold+- Test for array token positions**Search index**diff --git a/include/art.h b/include/art.hindex 89f55da9..976a47c5 100644--- a/include/art.h+++ b/include/art.h@@ -99,7 +99,7 @@ typedef struct {typedef struct {uint32_t key_len;int64_t max_score;- art_values* values;+ void* values;unsigned char key[];} art_leaf;@@ -118,8 +118,7 @@ typedef struct {typedef struct {int32_t score;uint32_t id;- uint32_t offsets_len;- uint32_t* offsets;+ std::vector<uint32_t> offsets;} art_document;enum token_ordering {@@ -186,7 +185,7 @@ inline uint64_t art_size(art_tree *t) {* @return NULL if the item was newly inserted, otherwise* the old value pointer is returned.*/-void* art_insert(art_tree *t, const unsigned char *key, int key_len, art_document* document, uint32_t num_hits);+void* art_insert(art_tree *t, const unsigned char *key, int key_len, art_document* document);/*** Deletes a value from the ART treediff --git a/include/index.h b/include/index.hindex fcd1057c..c123a92f 100644--- a/include/index.h+++ b/include/index.h@@ -20,6 +20,7 @@#include "num_tree.h"#include "magic_enum.hpp"#include "match_score.h"+#include "posting_list.h"struct token_t {size_t position;@@ -309,8 +310,10 @@ public:void score_results(const std::vector<sort_by> &sort_fields, const uint16_t &query_index, const uint8_t &field_id,const uint32_t total_cost, Topster *topster, const std::vector<art_leaf *> &query_suggestion,- spp::sparse_hash_set<uint64_t> &groups_processed, const uint32_t *result_ids,- const size_t result_size, const size_t group_limit,+ spp::sparse_hash_set<uint64_t> &groups_processed,+ const std::vector<std::unordered_map<size_t, std::vector<token_positions_t>>>& array_token_positions_vec,+ const uint32_t* result_ids, size_t result_ids_size,+ const size_t group_limit,const std::vector<std::string> &group_by_fields, uint32_t token_bits,const std::vector<token_t> &query_tokens, bool prioritize_exact_match) const;@@ -326,10 +329,6 @@ public:uint64_t get_distinct_id(const std::vector<std::string>& group_by_fields, const uint32_t seq_id) const;- void eq_str_filter_plain(const uint32_t *strt_ids, size_t strt_ids_size,- const std::vector<art_leaf *> &query_suggestion,- uint32_t *exact_strt_ids, size_t& exact_strt_size) const;-void scrub_reindex_doc(nlohmann::json& update_doc, nlohmann::json& del_doc, nlohmann::json& old_doc);static void tokenize_string_field(const nlohmann::json& document,@@ -374,16 +373,6 @@ public:const std::map<std::string, field> & facet_schema,const std::string& fallback_field_type);- static void populate_token_positions(const std::vector<art_leaf *> &query_suggestion,- const std::vector<uint32_t*>& leaf_to_indices,- const size_t result_index,- std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_positions);-- /*static bool is_exact_token_match(const art_leaf* leaf,- const std::vector<uint32_t*>& leaf_to_indices,- const size_t result_index,- std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_positions);-*/static bool is_point_in_polygon(const Geofence& poly, const GeoCoord& point);static double transform_for_180th_meridian(Geofence& poly);diff --git a/src/art.cpp b/src/art.cppindex 0114fcd4..0ff09564 100644--- a/src/art.cpp+++ b/src/art.cpp@@ -12,6 +12,7 @@#include <limits>#include <queue>#include <stdint.h>+#include <posting.h>#include "art.h"#include "logger.h"@@ -39,10 +40,8 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *void art_int_fuzzy_recurse(art_node *n, int depth, const unsigned char* int_str, int int_str_len,NUM_COMPARATOR comparator, std::vector<const art_leaf *> &results);-static void insert_and_shift_offset_index(sorted_array& offset_index, const uint32_t index, const uint32_t num_offsets);-bool compare_art_leaf_frequency(const art_leaf *a, const art_leaf *b) {- return a->values->ids.getLength() > b->values->ids.getLength();+ return posting_t::num_ids(a->values) > posting_t::num_ids(b->values);}bool compare_art_leaf_score(const art_leaf *a, const art_leaf *b) {@@ -54,12 +53,12 @@ bool compare_art_node_frequency(const art_node *a, const art_node *b) {if(IS_LEAF(a)) {art_leaf* al = (art_leaf *) LEAF_RAW(a);- a_value = al->values->ids.getLength();+ a_value = posting_t::num_ids(al->values);}if(IS_LEAF(b)) {art_leaf* bl = (art_leaf *) LEAF_RAW(b);- b_value = bl->values->ids.getLength();+ b_value = posting_t::num_ids(bl->values);}return a_value > b_value;@@ -138,7 +137,7 @@ static void destroy_node(art_node *n) {// Special case leafsif (IS_LEAF(n)) {art_leaf *leaf = (art_leaf *) LEAF_RAW(n);- delete leaf->values;+ posting_t::destroy_list(leaf->values);free(leaf);return;}@@ -415,47 +414,20 @@ art_leaf* art_maximum(art_tree *t) {static void add_document_to_leaf(const art_document *document, art_leaf *leaf) {leaf->max_score = MAX(leaf->max_score, document->score);- size_t inserted_index = leaf->values->ids.append(document->id);-- if(inserted_index == leaf->values->ids.getLength()-1) {- // treat as appends- uint32_t curr_index = leaf->values->offsets.getLength();- leaf->values->offset_index.append(curr_index);- for(uint32_t i=0; i<document->offsets_len; i++) {- leaf->values->offsets.append(document->offsets[i]);- }- } else {- uint32_t existing_offset_index = leaf->values->offset_index.at(inserted_index);- insert_and_shift_offset_index(leaf->values->offset_index, inserted_index, document->offsets_len);- leaf->values->offsets.insert(existing_offset_index, document->offsets, document->offsets_len);- }-}--void insert_and_shift_offset_index(sorted_array& offset_index, const uint32_t index, const uint32_t num_offsets) {- uint32_t existing_offset_index = offset_index.at(index);- uint32_t length = offset_index.getLength();- uint32_t new_length = length + 1;- uint32_t *curr_array = offset_index.uncompress(new_length);-- memmove(&curr_array[index+1], &curr_array[index], sizeof(uint32_t)*(length - index));- curr_array[index] = existing_offset_index;-- uint32_t curr_index = index + 1;- while(curr_index < new_length) {- curr_array[curr_index] += num_offsets;- curr_index++;- }-- offset_index.load(curr_array, new_length);-- delete [] curr_array;+ posting_t::upsert(leaf->values, document->id, document->offsets);}static art_leaf* make_leaf(const unsigned char *key, uint32_t key_len, art_document *document) {art_leaf *l = (art_leaf *) malloc(sizeof(art_leaf) + key_len);- l->values = new art_values;- l->max_score = 0;l->key_len = key_len;+ l->max_score = 0;++ uint32_t ids[1] = {document->id};+ uint32_t offset_index[1] = {0};+ compact_posting_list_t* list = compact_posting_list_t::create(1, ids, offset_index, document->offsets.size(),+ &document->offsets[0]);+ l->values = SET_COMPACT_POSTING(list);+memcpy(l->key, key, key_len);add_document_to_leaf(document, l);return l;@@ -622,7 +594,7 @@ static int prefix_mismatch(const art_node *n, const unsigned char *key, int key_return idx;}-static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *key, uint32_t key_len, art_document *document, uint32_t num_hits, int depth, int *old) {+static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *key, uint32_t key_len, art_document *document, int depth, int *old) {// If we are at a NULL node, inject a leafif (!n) {*ref = (art_node*)SET_LEAF(make_leaf(key, key_len, document));@@ -636,14 +608,8 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *// Check if we are updating an existing valueif (!leaf_matches(l, key, key_len, depth)) {*old = 1;- art_values *ret_val = l->values;-- // updates are not supported- if(!l->values->ids.contains(document->id)) {- add_document_to_leaf(document, l);- }-- return ret_val;+ add_document_to_leaf(document, l);+ return l->values;}// New value, we must split the leaf into a node4@@ -705,7 +671,7 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *// Find a child to recurse toart_node **child = find_child(n, key[depth]);if (child) {- return recursive_insert(*child, child, key, key_len, document, num_hits, depth + 1, old);+ return recursive_insert(*child, child, key, key_len, document, depth + 1, old);}// No child, node goes within us@@ -723,10 +689,10 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char ** @return NULL if the item was newly inserted, otherwise* the old value pointer is returned.*/-void* art_insert(art_tree *t, const unsigned char *key, int key_len, art_document* document, uint32_t num_hits) {+void* art_insert(art_tree *t, const unsigned char *key, int key_len, art_document* document) {int old_val = 0;- void *old = recursive_insert(t->root, &t->root, key, key_len, document, num_hits, 0, &old_val);+ void *old = recursive_insert(t->root, &t->root, key, key_len, document, 0, &old_val);if (!old_val) t->size++;return old;}@@ -951,8 +917,8 @@ int art_topk_iter(const art_node *root, token_ordering token_order, size_t max_rresults.push_back(l);} else {// we will push leaf only if filter matches with leaf IDs- size_t found_len = l->values->ids.numFoundOf(filter_ids, filter_ids_length);- if(found_len != 0) {+ bool found_atleast_one = posting_t::contains_atleast_one(l->values, filter_ids, filter_ids_length);+ if(found_atleast_one) {results.push_back(l);}}diff --git a/src/collection.cpp b/src/collection.cppindex 4414b39d..84b7bc70 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -15,6 +15,7 @@#include <h3api.h>#include <regex>#include <list>+#include <posting.h>#include "topster.h"#include "logger.h"@@ -1478,11 +1479,10 @@ void Collection::highlight_result(const field &search_field,return ;}- std::vector<uint32_t*> leaf_to_indices;std::vector<art_leaf*> query_suggestion;std::set<std::string> query_suggestion_tokens;- for (const art_leaf *token_leaf : searched_queries[field_order_kv->query_index]) {+ for (const art_leaf* token_leaf : searched_queries[field_order_kv->query_index]) {// Must search for the token string fresh on that field for the given document since `token_leaf`// is from the best matched field and need not be present in other fields of a document.Index* index = indices[field_order_kv->key % num_memory_shards];@@ -1494,11 +1494,6 @@ void Collection::highlight_result(const field &search_field,query_suggestion.push_back(actual_leaf);std::string token(reinterpret_cast<char*>(actual_leaf->key), actual_leaf->key_len-1);query_suggestion_tokens.insert(token);- std::vector<uint16_t> positions;- uint32_t doc_index = actual_leaf->values->ids.indexOf(field_order_kv->key);- auto doc_indices = new uint32_t[1];- doc_indices[0] = doc_index;- leaf_to_indices.push_back(doc_indices);}}@@ -1514,30 +1509,31 @@ void Collection::highlight_result(const field &search_field,reinterpret_cast<const unsigned char *>(q_token.c_str()),q_token.size() + 1);if(actual_leaf != nullptr) {- std::vector<uint16_t> positions;- uint32_t doc_index = actual_leaf->values->ids.indexOf(field_order_kv->key);- if(doc_index != actual_leaf->values->ids.getLength()) {- auto doc_indices = new uint32_t[1];- doc_indices[0] = doc_index;- leaf_to_indices.push_back(doc_indices);- query_suggestion.push_back(actual_leaf);- }+ query_suggestion.push_back(actual_leaf);}}}if(query_suggestion.empty()) {// none of the tokens from the query were found on this field- free_leaf_indices(leaf_to_indices);return ;}//LOG(INFO) << "Document ID: " << document["id"];- // positions in the field of each token in the query- std::unordered_map<size_t, std::vector<token_positions_t>> array_token_positions;- Index::populate_token_positions(query_suggestion, leaf_to_indices, 0, array_token_positions);+ std::vector<void*> posting_lists;+ for(art_leaf* leaf: query_suggestion) {+ posting_lists.push_back(leaf->values);+ }++ std::vector<std::unordered_map<size_t, std::vector<token_positions_t>>> array_token_positions_vec;+ posting_t::get_array_token_positions(field_order_kv->key, posting_lists, array_token_positions_vec);+ if(array_token_positions_vec.empty()) {+ return;+ }++ std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_positions = array_token_positions_vec[0];std::vector<match_index_t> match_indices;for(const auto& kv: array_token_positions) {@@ -1559,7 +1555,6 @@ void Collection::highlight_result(const field &search_field,if(match_indices.empty()) {// none of the tokens from the query were found on this field- free_leaf_indices(leaf_to_indices);return ;}@@ -1707,8 +1702,6 @@ void Collection::highlight_result(const field &search_field,highlight.field = search_field.name;highlight.match_score = match_indices[0].match_score;-- free_leaf_indices(leaf_to_indices);}void Collection::free_leaf_indices(std::vector<uint32_t*>& leaf_to_indices) const {diff --git a/src/index.cpp b/src/index.cppindex 93f157a8..5d6295ba 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -17,6 +17,7 @@#include <s2/s2earth.h>#include <s2/s2loop.h>#include <s2/s2builder.h>+#include <posting.h>#include "logger.h"Index::Index(const std::string name, const std::unordered_map<std::string, field> & search_schema,@@ -506,29 +507,15 @@ void Index::insert_doc(const int64_t score, art_tree *t, uint32_t seq_id,art_document art_doc;art_doc.id = seq_id;art_doc.score = score;- art_doc.offsets_len = (uint32_t) kv.second.size();- art_doc.offsets = new uint32_t[kv.second.size()];+ art_doc.offsets = kv.second;uint32_t num_hits = 0;const unsigned char *key = (const unsigned char *) kv.first.c_str();int key_len = (int) kv.first.length() + 1; // for the terminating \0 char- art_leaf* leaf = (art_leaf *) art_search(t, key, key_len);- if(leaf != NULL) {- num_hits = leaf->values->ids.getLength();- }-- num_hits += 1;-- for(size_t i=0; i<kv.second.size(); i++) {- art_doc.offsets[i] = kv.second[i];- }-//LOG(INFO) << "key: " << key << ", art_doc.id: " << art_doc.id;- art_insert(t, key, key_len, &art_doc, num_hits);- delete [] art_doc.offsets;- art_doc.offsets = nullptr;+ art_insert(t, key, key_len, &art_doc);}}@@ -929,95 +916,106 @@ void Index::search_candidates(const uint8_t & field_id,}*/// initialize results with the starting element (for further intersection)- size_t result_size = query_suggestion[0]->values->ids.getLength();+ size_t result_size = posting_t::num_ids(query_suggestion[0]->values);if(result_size == 0) {continue;}- uint32_t* result_ids = query_suggestion[0]->values->ids.uncompress();+ // Prepare excluded document IDs that we can later remove from the result set+ uint32_t* excluded_result_ids = nullptr;+ size_t excluded_result_ids_size = ArrayUtils::or_scalar(exclude_token_ids, exclude_token_ids_size,+ &curated_ids[0], curated_ids.size(), &excluded_result_ids);- // intersect the document ids for each token to find docs that contain all the tokens (stored in `result_ids`)- for(size_t i=1; i < query_suggestion.size(); i++) {- uint32_t* out = nullptr;- uint32_t* ids = query_suggestion[i]->values->ids.uncompress();- result_size = ArrayUtils::and_scalar(ids, query_suggestion[i]->values->ids.getLength(), result_ids, result_size, &out);- delete[] ids;- delete[] result_ids;- result_ids = out;- }+ std::vector<void*> posting_lists;- if(result_size == 0) {- delete[] result_ids;- continue;+ for(auto& query_leaf : query_suggestion) {+ posting_lists.push_back(query_leaf->values);}- // Exclude document IDs associated with excluded tokens from the result set- if(exclude_token_ids_size != 0) {- uint32_t *excluded_result_ids = nullptr;- result_size = ArrayUtils::exclude_scalar(result_ids, result_size, exclude_token_ids, exclude_token_ids_size,- &excluded_result_ids);- delete[] result_ids;- result_ids = excluded_result_ids;- }+ std::vector<posting_list_t::iterator_t> its;+ posting_list_t::result_iter_state_t iter_state;+ std::vector<uint32_t> result_id_vec;- if(!curated_ids.empty()) {- uint32_t *excluded_result_ids = nullptr;- result_size = ArrayUtils::exclude_scalar(result_ids, result_size, &curated_ids[0],- curated_ids.size(), &excluded_result_ids);+ size_t excluded_result_ids_index = 0;+ size_t filter_ids_index = 0;- delete [] result_ids;- result_ids = excluded_result_ids;- }+ posting_t::block_intersector_t intersector(posting_lists, 1000, iter_state);+ bool has_more = true;- //LOG(INFO) << "n: " << n;- /*std::stringstream log_query;- for(size_t i=0; i < query_suggestion.size(); i++) {- log_query << query_suggestion[i]->key << " ";- }*/+ while(has_more) {+ has_more = intersector.intersect();+ posting_list_t::result_iter_state_t updated_iter_state;- if(filter_ids != nullptr) {- // intersect once again with filter ids- uint32_t* filtered_result_ids = nullptr;- size_t filtered_results_size = ArrayUtils::and_scalar(filter_ids, filter_ids_length, result_ids,- result_size, &filtered_result_ids);+ for(size_t i = 0; i < iter_state.ids.size(); i++) {+ uint32_t id = iter_state.ids[i];- uint32_t* new_all_result_ids = nullptr;- all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, filtered_result_ids,- filtered_results_size, &new_all_result_ids);- delete [] *all_result_ids;- *all_result_ids = new_all_result_ids;+ // decide if this result id should be excluded+ if(excluded_result_ids_size != 0) {+ while(excluded_result_ids_index < excluded_result_ids_size &&+ excluded_result_ids[excluded_result_ids_index] < id) {+ excluded_result_ids_index++;+ }- // go through each matching document id and calculate match score- score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster,- query_suggestion,- groups_processed, filtered_result_ids, filtered_results_size,- group_limit, group_by_fields, token_bits, query_tokens, prioritize_exact_match);+ if(excluded_result_ids_index < excluded_result_ids_size &&+ id == excluded_result_ids[excluded_result_ids_index]) {+ excluded_result_ids_index++;+ continue;+ }+ }- field_num_results += filtered_results_size;+ bool id_found_in_filter = true;- delete[] filtered_result_ids;- delete[] result_ids;- } else {- uint32_t* new_all_result_ids = nullptr;- all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, result_ids,- result_size, &new_all_result_ids);- delete [] *all_result_ids;- *all_result_ids = new_all_result_ids;+ // decide if this result be matched with filter results+ if(filter_ids_length != 0) {+ id_found_in_filter = false;- /*if(result_size != 0) {- LOG(INFO) << size_t(field_id) << " - " << log_query.str() << ", result_size: " << result_size;- }*/+ // e.g. [1, 3] vs [2, 3]- score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster,- query_suggestion,- groups_processed, result_ids, result_size, group_limit, group_by_fields, token_bits,- query_tokens, prioritize_exact_match);+ while(filter_ids_index < filter_ids_length && filter_ids[filter_ids_index] < id) {+ filter_ids_index++;+ }- field_num_results += result_size;+ if(filter_ids_index < filter_ids_length && filter_ids[filter_ids_index] == id) {+ filter_ids_index++;+ id_found_in_filter = true;+ }+ }++ if(id_found_in_filter) {+ result_id_vec.push_back(id);- delete[] result_ids;+ updated_iter_state.ids.push_back(id);+ updated_iter_state.blocks.push_back(iter_state.blocks[i]);+ updated_iter_state.indices.push_back(iter_state.indices[i]);+ }+ }++ std::vector<std::unordered_map<size_t, std::vector<token_positions_t>>> array_token_positions_vec;+ posting_list_t::get_offsets(updated_iter_state, array_token_positions_vec);++ score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster,+ query_suggestion, groups_processed, array_token_positions_vec,+ &updated_iter_state.ids[0], updated_iter_state.ids.size(),+ group_limit, group_by_fields, token_bits, query_tokens, prioritize_exact_match);+ }++ if(result_id_vec.empty()) {+ continue;}+ uint32_t* result_ids = &result_id_vec[0];+ result_size = result_id_vec.size();++ field_num_results += result_id_vec.size();++ uint32_t* new_all_result_ids = nullptr;+ all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, result_ids,+ result_size, &new_all_result_ids);+ delete [] *all_result_ids;+ *all_result_ids = new_all_result_ids;++ delete [] excluded_result_ids;+searched_queries.push_back(actual_query_suggestion);//LOG(INFO) << "field_num_results: " << field_num_results << ", typo_tokens_threshold: " << typo_tokens_threshold;@@ -1179,7 +1177,7 @@ uint32_t Index::do_filtering(uint32_t** filter_ids_out, const std::vector<filteruint32_t* strt_ids = nullptr;size_t strt_ids_size = 0;- std::vector<art_leaf *> query_suggestion;+ std::vector<void*> posting_lists;// there could be multiple tokens in a filter value, which we have to treat as ANDs// e.g. country: South Africa@@ -1199,27 +1197,19 @@ uint32_t Index::do_filtering(uint32_t** filter_ids_out, const std::vector<filtercontinue;}- query_suggestion.push_back(leaf);+ posting_lists.push_back(leaf->values);}- if(query_suggestion.size() != str_tokens.size()) {+ if(posting_lists.size() != str_tokens.size()) {continue;}- for(const auto& leaf: query_suggestion) {- if(strt_ids == nullptr) {- strt_ids = leaf->values->ids.uncompress();- strt_ids_size = leaf->values->ids.getLength();- } else {- // do AND for an exact match- uint32_t* out = nullptr;- uint32_t* leaf_ids = leaf->values->ids.uncompress();- strt_ids_size = ArrayUtils::and_scalar(strt_ids, strt_ids_size, leaf_ids,- leaf->values->ids.getLength(), &out);- delete[] leaf_ids;- delete[] strt_ids;- strt_ids = out;- }+ std::vector<uint32_t> result_id_vec;+ posting_t::intersect(posting_lists, result_id_vec);+ if(!result_id_vec.empty()) {+ strt_ids = new uint32_t [result_id_vec.size()];+ std::copy(result_id_vec.begin(), result_id_vec.end(), strt_ids);+ strt_ids_size = result_id_vec.size();}if((a_filter.comparators[0] == EQUALS || a_filter.comparators[0] == NOT_EQUALS) && f.is_facet()) {@@ -1234,7 +1224,7 @@ uint32_t Index::do_filtering(uint32_t** filter_ids_out, const std::vector<filterbool found_filter = false;if(!f.is_array()) {- found_filter = (query_suggestion.size() == fvalues.length);+ found_filter = (posting_lists.size() == fvalues.length);} else {uint64_t filter_hash = 1;@@ -1337,59 +1327,6 @@ uint32_t Index::do_filtering(uint32_t** filter_ids_out, const std::vector<filterreturn filter_ids_length;}-void Index::eq_str_filter_plain(const uint32_t *strt_ids, size_t strt_ids_size,- const std::vector<art_leaf *>& query_suggestion, uint32_t *exact_strt_ids,- size_t& exact_strt_size) const {- std::vector<uint32_t*> leaf_to_indices;- for (art_leaf *token_leaf: query_suggestion) {- if(token_leaf == nullptr) {- leaf_to_indices.push_back(nullptr);- continue;- }-- uint32_t *indices = new uint32_t[strt_ids_size];- token_leaf->values->ids.indexOf(strt_ids, strt_ids_size, indices);- leaf_to_indices.push_back(indices);- }-- // e.g. First In First Out => hash([0, 1, 0, 2])- spp::sparse_hash_map<size_t, uint32_t> leaf_to_id;- size_t next_id = 1;- size_t filter_hash = 1;-- for(size_t leaf_index=0; leaf_index<query_suggestion.size(); leaf_index++) {- if(leaf_to_id.count(leaf_index) == 0) {- leaf_to_id.emplace(leaf_index, next_id++);- }-- uint32_t leaf_id = leaf_to_id[leaf_index];- filter_hash *= (1779033703 + 2*leaf_id*(leaf_index+1));- }-- for(size_t strt_ids_index = 0; strt_ids_index < strt_ids_size; strt_ids_index++) {- std::unordered_map<size_t, std::vector<token_positions_t>> array_token_positions;- populate_token_positions(query_suggestion, leaf_to_indices, strt_ids_index, array_token_positions);- // iterate array_token_positions and compute hash-- for(const auto& kv: array_token_positions) {- const std::vector<token_positions_t>& token_positions = kv.second;- size_t this_hash = 1;-- for(size_t token_index = 0; token_index < token_positions.size(); token_index++) {- auto& positions = token_positions[token_index].positions;- for(auto pos: positions) {- this_hash *= (1779033703 + 2*(token_index+1)*(pos+1));- }- }-- if(this_hash == filter_hash) {- exact_strt_ids[exact_strt_size++] = strt_ids[strt_ids_index];- break;- }- }- }-}-void Index::run_search(search_args* search_params) {search(search_params->field_query_tokens,search_params->search_fields,@@ -1535,22 +1472,30 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,// find documents that contain the excluded tokens to exclude them from results laterfor(size_t i = 0; i < num_search_fields; i++) {const std::string & field_name = search_fields[i].name;++ std::vector<void*> posting_lists;+for(const std::string& exclude_token: field_query_tokens[i].q_exclude_tokens) {art_leaf* leaf = (art_leaf *) art_search(search_index.at(field_name),(const unsigned char *) exclude_token.c_str(),exclude_token.size() + 1);if(leaf) {- uint32_t *ids = leaf->values->ids.uncompress();- uint32_t *exclude_token_ids_merged = nullptr;- exclude_token_ids_size = ArrayUtils::or_scalar(exclude_token_ids, exclude_token_ids_size, ids,- leaf->values->ids.getLength(),- &exclude_token_ids_merged);- delete[] ids;- delete[] exclude_token_ids;- exclude_token_ids = exclude_token_ids_merged;+ posting_lists.push_back(leaf->values);}}++ std::vector<uint32_t> exclude_token_id_vec;+ if(!posting_lists.empty()) {+ posting_t::merge(posting_lists, exclude_token_id_vec);+ }++ uint32_t *exclude_token_ids_merged = nullptr;+ exclude_token_ids_size = ArrayUtils::or_scalar(exclude_token_ids, exclude_token_ids_size,+ &exclude_token_id_vec[0], exclude_token_id_vec.size(),+ &exclude_token_ids_merged);+ delete[] exclude_token_ids;+ exclude_token_ids = exclude_token_ids_merged;}std::vector<Topster*> ftopsters;@@ -1599,7 +1544,7 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,uint32_t token_bits = 255;score_results(sort_fields_std, (uint16_t) searched_queries.size(), field_id, 0, topster, {},- groups_processed, filter_ids, filter_ids_length, group_limit, group_by_fields, token_bits, {},+ groups_processed, {}, filter_ids, filter_ids_length, group_limit, group_by_fields, token_bits, {},prioritize_exact_match);collate_included_ids(field_query_tokens[0].q_include_tokens, field, field_id, included_ids_map, curated_topster, searched_queries);@@ -1752,8 +1697,7 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,continue;}- uint32_t doc_index = leaves[0]->values->ids.indexOf(seq_id);- if (doc_index == leaves[0]->values->ids.getLength()) {+ if(!posting_t::contains(leaves[0]->values, seq_id)) {continue;}@@ -2016,8 +1960,8 @@ void Index::log_leaves(const int cost, const std::string &token, const std::vectfor(size_t i=0; i < leaves.size(); i++) {std::string key((char*)leaves[i]->key, leaves[i]->key_len);- LOG(INFO) << key << " - " << leaves[i]->values->ids.getLength();- LOG(INFO) << "frequency: " << leaves[i]->values->ids.getLength() << ", max_score: " << leaves[i]->max_score;+ LOG(INFO) << key << " - " << posting_t::num_ids(leaves[i]->values);+ LOG(INFO) << "frequency: " << posting_t::num_ids(leaves[i]->values) << ", max_score: " << leaves[i]->max_score;/*for(auto j=0; j<leaves[i]->values->ids.getLength(); j++) {LOG(INFO) << "id: " << leaves[i]->values->ids.at(j);}*/@@ -2028,7 +1972,8 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16const uint8_t & field_id, const uint32_t total_cost, Topster* topster,const std::vector<art_leaf *> &query_suggestion,spp::sparse_hash_set<uint64_t>& groups_processed,- const uint32_t *result_ids, const size_t result_size,+ const std::vector<std::unordered_map<size_t, std::vector<token_positions_t>>>& array_token_positions_vec,+ const uint32_t* result_ids, size_t result_ids_size,const size_t group_limit, const std::vector<std::string>& group_by_fields,uint32_t token_bits,const std::vector<token_t>& query_tokens,@@ -2060,7 +2005,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16S2LatLng reference_lat_lng;GeoPoint::unpack_lat_lng(sort_fields[i].geopoint, reference_lat_lng);- for (size_t rindex = 0; rindex < result_size; rindex++) {+ for (size_t rindex = 0; rindex < result_ids_size; rindex++) {const uint32_t seq_id = result_ids[rindex];auto it = geopoints->find(seq_id);int64_t dist = INT32_MAX;@@ -2085,13 +2030,6 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16}}- std::vector<uint32_t *> leaf_to_indices;- for (art_leaf *token_leaf: query_suggestion) {- uint32_t *indices = new uint32_t[result_size];- token_leaf->values->ids.indexOf(result_ids, result_size, indices);- leaf_to_indices.push_back(indices);- }-Match single_token_match = Match(1, 0);const uint64_t single_token_match_score = single_token_match.get_match_score(total_cost);@@ -2103,16 +2041,16 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16(query_suggestion.size() == query_tokens.size()) &&((std::string((const char*)query_suggestion[0]->key, query_suggestion[0]->key_len-1) != query_tokens[0].value));- for (size_t i = 0; i < result_size; i++) {+ for (size_t i = 0; i < result_ids_size; i++) {const uint32_t seq_id = result_ids[i];uint64_t match_score = 0;- if (use_single_token_score) {+ if (use_single_token_score || array_token_positions_vec.empty()) {match_score = single_token_match_score;} else {- std::unordered_map<size_t, std::vector<token_positions_t>> array_token_positions;- populate_token_positions(query_suggestion, leaf_to_indices, i, array_token_positions);+ const std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_positions =+ array_token_positions_vec[i];for (const auto& kv: array_token_positions) {const std::vector<token_positions_t>& token_positions = kv.second;@@ -2201,10 +2139,6 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16//long long int timeNanos = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();//LOG(INFO) << "Time taken for results iteration: " << timeNanos << "ms";-- for(uint32_t* leaf_indices: leaf_to_indices) {- delete [] leaf_indices;- }}// pre-filter group_by_fields such that we can avoid the find() check@@ -2236,103 +2170,6 @@ uint64_t Index::get_distinct_id(const std::vector<std::string>& group_by_fields,return distinct_id;}-void Index::populate_token_positions(const std::vector<art_leaf *>& query_suggestion,- const std::vector<uint32_t*>& leaf_to_indices,- const size_t result_index,- std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_positions) {- if(query_suggestion.empty()) {- return ;- }-- // array_token_positions:- // for every element in a potential array, for every token in query suggestion, get the positions-- for(size_t i = 0; i < query_suggestion.size(); i++) {- const art_leaf* token_leaf = query_suggestion[i];- uint32_t doc_index = leaf_to_indices[i][result_index];- /*LOG(INFO) << "doc_id: " << token_leaf->values->ids.at(doc_index) << ", token_leaf->values->ids.getLength(): "- << token_leaf->values->ids.getLength();*/-- // it's possible for a query token to not appear in a resulting document- if(doc_index == token_leaf->values->ids.getLength()) {- continue;- }-- //LOG(INFO) << "key: " << token_leaf->key;-- // Plain string format:- // offset1, offset2, ... , 0 (if token is the last offset for the document)-- // Array string format:- // offset1, ... , offsetn, offsetn, array_index, 0 (if token is the last offset for the document)- // (last offset is repeated to indicate end of offsets for a given array index)-- /*uint32_t* offsets = token_leaf->values->offsets.uncompress();- for(size_t ii=0; ii < token_leaf->values->offsets.getLength(); ii++) {- LOG(INFO) << "offset: " << offsets[ii];- }-- uint32_t* offset_indices = token_leaf->values->offset_index.uncompress();- for(size_t ii=0; ii < token_leaf->values->offset_index.getLength(); ii++) {- LOG(INFO) << "offset index: " << offset_indices[ii];- }-- LOG(INFO) << "token_leaf->values->offsets.getLength(): " << token_leaf->values->offsets.getLength();*/-- uint32_t start_offset = token_leaf->values->offset_index.at(doc_index);- uint32_t end_offset = (doc_index == token_leaf->values->ids.getLength() - 1) ?- token_leaf->values->offsets.getLength() :- token_leaf->values->offset_index.at(doc_index+1);-- std::vector<uint16_t> positions;- int prev_pos = -1;- bool is_last_token = false;-- while(start_offset < end_offset) {- int pos = token_leaf->values->offsets.at(start_offset);- start_offset++;-- if(pos == 0) {- // indicates that token is the last token on the doc- is_last_token = true;- start_offset++;- continue;- }-- if(pos == prev_pos) { // indicates end of array index- if(!positions.empty()) {- size_t array_index = (size_t) token_leaf->values->offsets.at(start_offset);- is_last_token = false;-- if(start_offset+1 < end_offset) {- size_t next_offset = (size_t) token_leaf->values->offsets.at(start_offset + 1);- if(next_offset == 0) {- // indicates that token is the last token on the doc- is_last_token = true;- start_offset++;- }- }-- array_token_positions[array_index].push_back(token_positions_t{is_last_token, positions});- positions.clear();- }-- start_offset++; // skip current value which is the array index or flag for last index- prev_pos = -1;- continue;- }-- prev_pos = pos;- positions.push_back((uint16_t)pos - 1);- }-- if(!positions.empty()) {- // for plain string fields- array_token_positions[0].push_back(token_positions_t{is_last_token, positions});- }- }-}-inline uint32_t Index::next_suggestion(const std::vector<token_candidates> &token_candidates_vec,long long int n,std::vector<art_leaf *>& actual_query_suggestion,@@ -2359,7 +2196,7 @@ inline uint32_t Index::next_suggestion(const std::vector<token_candidates> &toke// Sort ascending based on matched documents for each token for faster intersection.// However, this causes the token order to deviate from original query's order.sort(query_suggestion.begin(), query_suggestion.end(), [](const art_leaf* left, const art_leaf* right) {- return left->values->ids.getLength() < right->values->ids.getLength();+ return posting_t::num_ids(left->values) < posting_t::num_ids(right->values);});return total_cost;@@ -2419,39 +2256,17 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & docstd::vector<std::string> tokens;tokenize_string_field(document, search_field, tokens, search_field.locale);- for(auto & token: tokens) {+ for(size_t i = 0; i < tokens.size(); i++) {+ const auto& token = tokens[i];const unsigned char *key = (const unsigned char *) token.c_str();int key_len = (int) (token.length() + 1);art_leaf* leaf = (art_leaf *) art_search(search_index.at(field_name), key, key_len);if(leaf != nullptr) {- uint32_t doc_index = leaf->values->ids.indexOf(seq_id);-- if (doc_index == leaf->values->ids.getLength()) {- // not found - happens when 2 tokens repeat in a field, e.g "is it or is is not?"- continue;- }-- uint32_t start_offset = leaf->values->offset_index.at(doc_index);- uint32_t end_offset = (doc_index == leaf->values->ids.getLength() - 1) ?- leaf->values->offsets.getLength() :- leaf->values->offset_index.at(doc_index + 1);-- uint32_t doc_indices[1] = {doc_index};- remove_and_shift_offset_index(leaf->values->offset_index, doc_indices, 1);-- leaf->values->offsets.remove_index(start_offset, end_offset);- leaf->values->ids.remove_value(seq_id);-- /*len = leaf->values->offset_index.getLength();- for(auto i=0; i<len; i++) {- LOG(INFO) << "i: " << i << ", val: " << leaf->values->offset_index.at(i);- }- LOG(INFO) << "----";*/-- if (leaf->values->ids.getLength() == 0) {- art_values *values = (art_values *) art_delete(search_index.at(field_name), key, key_len);- delete values;+ posting_t::erase(leaf->values, seq_id);+ if (posting_t::num_ids(leaf->values) == 0) {+ void* values = art_delete(search_index.at(field_name), key, key_len);+ posting_t::destroy_list(values);}}}diff --git a/test/art_test.cpp b/test/art_test.cppindex f81a9330..157b58dc 100644--- a/test/art_test.cpp+++ b/test/art_test.cpp@@ -6,6 +6,7 @@#include <gtest/gtest.h>#include <art.h>#include <chrono>+#include <posting.h>#define words_file_path std::string(std::string(ROOT_DIR)+"/build/test_resources/words.txt").c_str()#define uuid_file_path std::string(std::string(ROOT_DIR)+"/build/test_resources/uuid.txt").c_str()@@ -16,9 +17,7 @@ art_document get_document(uint32_t id) {art_document document;document.score = id;document.id = id;- document.offsets = new uint32_t[1]{0};- document.offsets_len = 1;-+ document.offsets = {0};return document;}@@ -48,11 +47,9 @@ TEST(ArtTest, test_art_insert) {len = strlen(buf);buf[len-1] = '\0';art_document document = get_document(line);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)buf, len, &document, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)buf, len, &document));ASSERT_TRUE(art_size(&t) == line);line++;-- delete [] document.offsets;}res = art_tree_destroy(&t);@@ -105,9 +102,9 @@ TEST(ArtTest, test_art_insert_verylong) {art_document doc1 = get_document(1);art_document doc2 = get_document(2);- ASSERT_TRUE(NULL == art_insert(&t, key1, 299, &doc1, 1));- ASSERT_TRUE(NULL == art_insert(&t, key2, 302, &doc2, 2));- art_insert(&t, key2, 302, &doc2, 2);+ ASSERT_TRUE(NULL == art_insert(&t, key1, 299, &doc1));+ ASSERT_TRUE(NULL == art_insert(&t, key2, 302, &doc2));+ art_insert(&t, key2, 302, &doc2);EXPECT_EQ(art_size(&t), 2);res = art_tree_destroy(&t);@@ -128,7 +125,7 @@ TEST(ArtTest, test_art_insert_search) {len = strlen(buf);buf[len-1] = '\0';art_document doc = get_document((uint32_t) line);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)buf, len, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)buf, len, &doc));line++;}@@ -142,7 +139,7 @@ TEST(ArtTest, test_art_insert_search) {buf[len-1] = '\0';art_leaf* l = (art_leaf *) art_search(&t, (unsigned char*)buf, len);- EXPECT_EQ(line, l->values->ids.at(0));+ EXPECT_EQ(line, posting_t::first_id(l->values));line++;}@@ -173,7 +170,7 @@ TEST(ArtTest, test_art_insert_delete) {len = strlen(buf);buf[len-1] = '\0';art_document doc = get_document((uint32_t) line);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)buf, len, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)buf, len, &doc));line++;}@@ -191,12 +188,12 @@ TEST(ArtTest, test_art_insert_delete) {// Search first, ensure all entries still// visibleart_leaf* l = (art_leaf *) art_search(&t, (unsigned char*)buf, len);- EXPECT_EQ(line, l->values->ids.at(0));+ EXPECT_EQ(line, posting_t::first_id(l->values));// Delete, should get lineno back- art_values* values = (art_values*)art_delete(&t, (unsigned char*)buf, len);- EXPECT_EQ(line, values->ids.at(0));- delete values;+ void* values = (art_values*)art_delete(&t, (unsigned char*)buf, len);+ EXPECT_EQ(line, posting_t::first_id(values));+ posting_t::destroy_list(values);// Check the sizeASSERT_TRUE(art_size(&t) == nlines - line);@@ -214,7 +211,7 @@ TEST(ArtTest, test_art_insert_delete) {int iter_cb(void *data, const unsigned char* key, uint32_t key_len, void *val) {uint64_t *out = (uint64_t*)data;- uintptr_t line = ((art_values*)val)->ids.at(0);+ uintptr_t line = posting_t::first_id(val);uint64_t mask = (line * (key[0] + key_len));out[0]++;out[1] ^= mask;@@ -222,35 +219,35 @@ int iter_cb(void *data, const unsigned char* key, uint32_t key_len, void *val) {}TEST(ArtTest, test_art_insert_iter) {- art_tree t;- int res = art_tree_init(&t);- ASSERT_TRUE(res == 0);+ art_tree t;+ int res = art_tree_init(&t);+ ASSERT_TRUE(res == 0);- int len;- char buf[512];- FILE *f = fopen(words_file_path, "r");+ int len;+ char buf[512];+ FILE *f = fopen(words_file_path, "r");- uint64_t xor_mask = 0;- uintptr_t line = 1, nlines;- while (fgets(buf, sizeof buf, f)) {- len = strlen(buf);- buf[len-1] = '\0';- art_document doc = get_document((uint32_t) line);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)buf, len, &doc, 1));+ uint64_t xor_mask = 0;+ uintptr_t line = 1, nlines;+ while (fgets(buf, sizeof buf, f)) {+ len = strlen(buf);+ buf[len-1] = '\0';+ art_document doc = get_document((uint32_t) line);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)buf, len, &doc));- xor_mask ^= (line * (buf[0] + len));- line++;- }- nlines = line - 1;+ xor_mask ^= (line * (buf[0] + len));+ line++;+ }+ nlines = line - 1;- uint64_t out[] = {0, 0};- ASSERT_TRUE(art_iter(&t, iter_cb, &out) == 0);+ uint64_t out[] = {0, 0};+ ASSERT_TRUE(art_iter(&t, iter_cb, &out) == 0);- ASSERT_TRUE(out[0] == nlines);- ASSERT_TRUE(out[1] == xor_mask);+ ASSERT_TRUE(out[0] == nlines);+ ASSERT_TRUE(out[1] == xor_mask);- res = art_tree_destroy(&t);- ASSERT_TRUE(res == 0);+ res = art_tree_destroy(&t);+ ASSERT_TRUE(res == 0);}@@ -275,27 +272,27 @@ TEST(ArtTest, test_art_iter_prefix) {const char *s = "api.foo.bar";art_document doc = get_document((uint32_t) 1);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc));s = "api.foo.baz";doc = get_document((uint32_t) 2);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc));s = "api.foe.fum";doc = get_document((uint32_t) 3);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc));s = "abc.123.456";doc = get_document((uint32_t) 4);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc));s = "api.foo";doc = get_document((uint32_t) 5);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc));s = "api";doc = get_document((uint32_t) 6);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc));// Iterate over apiconst char *expected[] = {"api", "api.foe.fum", "api.foo", "api.foo.bar", "api.foo.baz"};@@ -352,27 +349,27 @@ TEST(ArtTest, test_art_long_prefix) {s = "this:key:has:a:long:prefix:3";id = 3;art_document doc = get_document((uint32_t) id);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc));s = "this:key:has:a:long:common:prefix:2";id = 2;doc = get_document((uint32_t) id);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc));s = "this:key:has:a:long:common:prefix:1";id = 1;doc = get_document((uint32_t) id);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc));// Search for the keyss = "this:key:has:a:long:common:prefix:1";- EXPECT_EQ(1, (((art_leaf *)art_search(&t, (unsigned char*)s, strlen(s)+1))->values->ids.at(0)));+ EXPECT_EQ(1, posting_t::first_id(((art_leaf *)art_search(&t, (unsigned char*)s, strlen(s)+1))->values));s = "this:key:has:a:long:common:prefix:2";- EXPECT_EQ(2, (((art_leaf *)art_search(&t, (unsigned char*)s, strlen(s)+1))->values->ids.at(0)));+ EXPECT_EQ(2, posting_t::first_id(((art_leaf *)art_search(&t, (unsigned char*)s, strlen(s)+1))->values));s = "this:key:has:a:long:prefix:3";- EXPECT_EQ(3, (((art_leaf *)art_search(&t, (unsigned char*)s, strlen(s)+1))->values->ids.at(0)));+ EXPECT_EQ(3, posting_t::first_id(((art_leaf *)art_search(&t, (unsigned char*)s, strlen(s)+1))->values));const char *expected[] = {"this:key:has:a:long:common:prefix:1",@@ -401,7 +398,7 @@ TEST(ArtTest, test_art_insert_search_uuid) {len = strlen(buf);buf[len-1] = '\0';art_document doc = get_document((uint32_t) line);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)buf, len, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)buf, len, &doc));line++;}@@ -414,7 +411,7 @@ TEST(ArtTest, test_art_insert_search_uuid) {len = strlen(buf);buf[len-1] = '\0';- uintptr_t id = ((art_leaf*)art_search(&t, (unsigned char*)buf, len))->values->ids.at(0);+ uintptr_t id = posting_t::first_id(((art_leaf*)art_search(&t, (unsigned char*)buf, len))->values);ASSERT_TRUE(line == id);line++;}@@ -438,15 +435,15 @@ TEST(ArtTest, test_art_max_prefix_len_scan_prefix) {const char* key1 = "foobarbaz1-test1-foo";art_document doc = get_document((uint32_t) 1);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key1, strlen(key1)+1, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key1, strlen(key1)+1, &doc));const char *key2 = "foobarbaz1-test1-bar";doc = get_document((uint32_t) 2);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key2, strlen(key2)+1, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key2, strlen(key2)+1, &doc));const char *key3 = "foobarbaz1-test2-foo";doc = get_document((uint32_t) 3);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key3, strlen(key3)+1, &doc, 2));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key3, strlen(key3)+1, &doc));ASSERT_TRUE(art_size(&t) == 3);@@ -469,15 +466,15 @@ TEST(ArtTest, test_art_prefix_iter_out_of_bounds) {const char* key1 = "foobarbaz1-long-test1-foo";art_document doc = get_document((uint32_t) 1);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key1, strlen(key1)+1, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key1, strlen(key1)+1, &doc));const char *key2 = "foobarbaz1-long-test1-bar";doc = get_document((uint32_t) 2);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key2, strlen(key2)+1, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key2, strlen(key2)+1, &doc));const char *key3 = "foobarbaz1-long-test2-foo";doc = get_document((uint32_t) 3);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key3, strlen(key3)+1, &doc, 2));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key3, strlen(key3)+1, &doc));ASSERT_TRUE(art_size(&t) == 3);@@ -500,15 +497,15 @@ TEST(ArtTest, test_art_search_out_of_bounds) {const char* key1 = "foobarbaz1-long-test1-foo";art_document doc = get_document((uint32_t) 1);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key1, strlen(key1)+1, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key1, strlen(key1)+1, &doc));const char *key2 = "foobarbaz1-long-test1-bar";doc = get_document((uint32_t) 2);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key2, strlen(key2)+1, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key2, strlen(key2)+1, &doc));const char *key3 = "foobarbaz1-long-test2-foo";doc = get_document((uint32_t) 3);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key3, strlen(key3)+1, &doc, 2));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key3, strlen(key3)+1, &doc));ASSERT_TRUE(art_size(&t) == 3);@@ -529,15 +526,15 @@ TEST(ArtTest, test_art_delete_out_of_bounds) {const char* key1 = "foobarbaz1-long-test1-foo";art_document doc = get_document((uint32_t) 1);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key1, strlen(key1)+1, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key1, strlen(key1)+1, &doc));const char *key2 = "foobarbaz1-long-test1-bar";doc = get_document((uint32_t) 2);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key2, strlen(key2)+1, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key2, strlen(key2)+1, &doc));const char *key3 = "foobarbaz1-long-test2-foo";doc = get_document((uint32_t) 3);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key3, strlen(key3)+1, &doc, 2));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key3, strlen(key3)+1, &doc));ASSERT_TRUE(art_size(&t) == 3);@@ -557,20 +554,24 @@ TEST(ArtTest, test_art_insert_multiple_ids_for_same_token) {const char* key1 = "implement";art_document doc = get_document((uint32_t) 1);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key1, strlen(key1)+1, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key1, strlen(key1)+1, &doc));doc = get_document((uint32_t) 2);- art_values* value = (art_values*) art_insert(&t, (unsigned char*)key1, strlen(key1) + 1, &doc, 2);+ void* value = art_insert(&t, (unsigned char*)key1, strlen(key1) + 1, &doc);ASSERT_TRUE(value != NULL);+ ASSERT_EQ(posting_t::num_ids(value), 2);+ ASSERT_EQ(posting_t::first_id(value), 1);+ ASSERT_TRUE(posting_t::contains(value, 2));+doc = get_document((uint32_t) 3);- ASSERT_TRUE(value == (art_values*) art_insert(&t, (unsigned char*)key1, strlen(key1)+1, &doc, 3));+ void* reinsert_value = art_insert(&t, (unsigned char*) key1, strlen(key1) + 1, &doc);ASSERT_TRUE(art_size(&t) == 1);- ASSERT_EQ(value->ids.getLength(), 3);- ASSERT_EQ(value->ids.at(0), 1);- ASSERT_EQ(value->ids.at(1), 2);- ASSERT_EQ(value->ids.at(2), 3);+ ASSERT_EQ(posting_t::num_ids(reinsert_value), 3);+ ASSERT_EQ(posting_t::first_id(reinsert_value), 1);+ ASSERT_TRUE(posting_t::contains(reinsert_value, 2));+ ASSERT_TRUE(posting_t::contains(reinsert_value, 3));res = art_tree_destroy(&t);ASSERT_TRUE(res == 0);@@ -583,10 +584,10 @@ TEST(ArtTest, test_art_fuzzy_search_single_leaf) {const char* implement_key = "implement";art_document doc = get_document((uint32_t) 1);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)implement_key, strlen(implement_key)+1, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)implement_key, strlen(implement_key)+1, &doc));art_leaf* l = (art_leaf *) art_search(&t, (const unsigned char *)implement_key, strlen(implement_key)+1);- EXPECT_EQ(1, l->values->ids.at(0));+ EXPECT_EQ(1, posting_t::first_id(l->values));std::vector<art_leaf*> leaves;art_fuzzy_search(&t, (const unsigned char *) implement_key, strlen(implement_key) + 1, 0, 0, 10, FREQUENCY, false, nullptr, 0, leaves);@@ -618,10 +619,10 @@ TEST(ArtTest, test_art_fuzzy_search_single_leaf_prefix) {const char* key = "application";art_document doc = get_document((uint32_t) 1);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key, strlen(key)+1, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key, strlen(key)+1, &doc));art_leaf* l = (art_leaf *) art_search(&t, (const unsigned char *)key, strlen(key)+1);- EXPECT_EQ(1, l->values->ids.at(0));+ EXPECT_EQ(1, posting_t::first_id(l->values));std::vector<art_leaf*> leaves;art_fuzzy_search(&t, (const unsigned char *) "aplication", strlen(key)-1, 0, 1, 10, FREQUENCY, true, nullptr, 0, leaves);@@ -649,7 +650,7 @@ TEST(ArtTest, test_art_fuzzy_search) {len = strlen(buf);buf[len-1] = '\0';art_document doc = get_document((uint32_t) line);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)buf, len, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)buf, len, &doc));line++;}@@ -757,12 +758,12 @@ TEST(ArtTest, test_art_fuzzy_search_unicode_chars) {for(const char* key: keys) {art_document doc = get_document((uint32_t) 1);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key, strlen(key)+1, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key, strlen(key)+1, &doc));}for(const char* key: keys) {art_leaf* l = (art_leaf *) art_search(&t, (const unsigned char *)key, strlen(key)+1);- EXPECT_EQ(1, l->values->ids.at(0));+ EXPECT_EQ(1, posting_t::first_id(l->values));std::vector<art_leaf*> leaves;art_fuzzy_search(&t, (unsigned char *)key, strlen(key), 0, 0, 10, FREQUENCY, true, nullptr, 0, leaves);@@ -789,7 +790,7 @@ TEST(ArtTest, test_art_search_sku_like_tokens) {len = strlen(buf);buf[len - 1] = '\0';art_document doc = get_document((uint32_t) line);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) buf, len, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) buf, len, &doc));keys.push_back(std::string(buf, len-1));line++;}@@ -798,7 +799,7 @@ TEST(ArtTest, test_art_search_sku_like_tokens) {// exact searchart_leaf* l = (art_leaf *) art_search(&t, (const unsigned char *)key1, strlen(key1)+1);- EXPECT_EQ(1, l->values->ids.getLength());+ EXPECT_EQ(1, posting_t::num_ids(l->values));// exact search all tokens via fuzzy API@@ -838,7 +839,7 @@ TEST(ArtTest, test_art_search_ill_like_tokens) {len = strlen(buf);buf[len - 1] = '\0';art_document doc = get_document((uint32_t) line);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) buf, len, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) buf, len, &doc));keys.push_back(std::string(buf, len-1));line++;}@@ -860,7 +861,7 @@ TEST(ArtTest, test_art_search_ill_like_tokens) {for (const auto &key : keys) {art_leaf* l = (art_leaf *) art_search(&t, (const unsigned char *)key.c_str(), key.size()+1);ASSERT_FALSE(l == nullptr);- EXPECT_EQ(1, l->values->ids.getLength());+ EXPECT_EQ(1, posting_t::num_ids(l->values));std::vector<art_leaf *> leaves;art_fuzzy_search(&t, (const unsigned char*)key.c_str(), key.size(), 0, 0, 10,@@ -895,18 +896,18 @@ TEST(ArtTest, test_art_search_ill_like_tokens2) {keys = {"input", "illustrations", "illustration"};art_document doc = get_document((uint32_t) 1);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) keys[0].c_str(), keys[0].size()+1, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) keys[0].c_str(), keys[0].size()+1, &doc));doc = get_document((uint32_t) 2);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) keys[1].c_str(), keys[1].size()+1, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) keys[1].c_str(), keys[1].size()+1, &doc));doc = get_document((uint32_t) 3);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) keys[2].c_str(), keys[2].size()+1, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) keys[2].c_str(), keys[2].size()+1, &doc));for (const auto &key : keys) {art_leaf* l = (art_leaf *) art_search(&t, (const unsigned char *)key.c_str(), key.size()+1);ASSERT_FALSE(l == nullptr);- EXPECT_EQ(1, l->values->ids.getLength());+ EXPECT_EQ(1, posting_t::num_ids(l->values));std::vector<art_leaf *> leaves;art_fuzzy_search(&t, (const unsigned char*)key.c_str(), key.size(), 0, 0, 10,@@ -941,7 +942,7 @@ TEST(ArtTest, test_art_search_roche_chews) {keys = {"roche"};art_document doc = get_document((uint32_t) 1);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) keys[0].c_str(), keys[0].size()+1, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) keys[0].c_str(), keys[0].size()+1, &doc));std::string term = "chews";std::vector<art_leaf *> leaves;@@ -969,7 +970,7 @@ TEST(ArtTest, test_art_search_raspberry) {for (const auto &key : keys) {art_document doc = get_document((uint32_t) 1);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) key.c_str(), key.size()+1, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) key.c_str(), key.size()+1, &doc));}// prefix search@@ -1002,7 +1003,7 @@ TEST(ArtTest, test_art_search_highliving) {for (const auto &key : keys) {art_document doc = get_document((uint32_t) 1);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) key.c_str(), key.size()+1, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) key.c_str(), key.size()+1, &doc));}// prefix search@@ -1086,7 +1087,7 @@ TEST(ArtTest, test_int32_overlap) {for(size_t j = 0; j < values[i].size(); j++) {encode_int32(values[i][j], chars);doc.id = i;- art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc, 1);+ art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc);}}@@ -1110,7 +1111,7 @@ TEST(ArtTest, test_int32_range_hundreds) {for(uint32_t i = 100; i < 110; i++) {encode_int32(i, chars);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc));}encode_int32(106, chars);@@ -1156,7 +1157,7 @@ TEST(ArtTest, test_int32_duplicates) {doc.id = i;int value = 1900 + (rand() % static_cast<int>(2018 - 1900 + 1));encode_int32(value, chars);- art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc, 1);+ art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc);}std::vector<const art_leaf*> results;@@ -1166,7 +1167,7 @@ TEST(ArtTest, test_int32_duplicates) {size_t counter = 0;for(auto res: results) {- counter += res->values->ids.getLength();+ counter += posting_t::num_ids(res->values);}ASSERT_EQ(10000, counter);@@ -1183,7 +1184,7 @@ TEST(ArtTest, test_int32_negative) {for(int32_t i = -100; i < 0; i++) {encode_int32(i, chars);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc));}encode_int32(-99, chars);@@ -1235,7 +1236,7 @@ TEST(ArtTest, test_int32_million) {for(uint32_t i = 0; i < 1000000; i++) {encode_int32(i, chars);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc));}encode_int32(5, chars);@@ -1314,7 +1315,7 @@ TEST(ArtTest, test_int_range_byte_boundary) {for(uint32_t i = 200; i < 300; i++) {encode_int32(i, chars);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc));}encode_int32(255, chars);@@ -1376,7 +1377,7 @@ TEST(ArtTest, test_search_int64) {for(uint64_t i = lmax; i < lmax+100; i++) {encode_int64(i, chars);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc));}std::vector<const art_leaf*> results;@@ -1419,7 +1420,7 @@ TEST(ArtTest, test_search_negative_int64) {for(int64_t i = lmax-100; i < lmax; i++) {encode_int64(i, chars);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc));}std::vector<const art_leaf*> results;@@ -1458,10 +1459,8 @@ TEST(ArtTest, test_search_negative_int64_large) {const int CHAR_LEN = 8;unsigned char chars[CHAR_LEN];- const int64_t lmax = -1 * std::numeric_limits<std::int32_t>::max();-encode_int64(-2, chars);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) chars, CHAR_LEN, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) chars, CHAR_LEN, &doc));std::vector<const art_leaf *> results;@@ -1491,7 +1490,7 @@ TEST(ArtTest, test_int32_array) {for (size_t j = 0; j < values[i].size(); j++) {encode_int32(values[i][j], chars);doc.id = i;- art_insert(&t, (unsigned char *) chars, CHAR_LEN, &doc, 1);+ art_insert(&t, (unsigned char *) chars, CHAR_LEN, &doc);}}@@ -1511,7 +1510,7 @@ TEST(ArtTest, test_encode_float_positive) {unsigned char chars0[CHAR_LEN];encode_float(floats[i], chars0);art_document doc = get_document(i);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars0, CHAR_LEN, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars0, CHAR_LEN, &doc));}std::vector<const art_leaf*> results;@@ -1568,7 +1567,7 @@ TEST(ArtTest, test_encode_float_positive_negative) {unsigned char chars0[CHAR_LEN];encode_float(floats[i], chars0);art_document doc = get_document(i);- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars0, CHAR_LEN, &doc, 1));+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars0, CHAR_LEN, &doc));}std::vector<const art_leaf*> results;diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cppindex 9f77ac27..e6d7f17c 100644--- a/test/collection_all_fields_test.cpp+++ b/test/collection_all_fields_test.cpp@@ -505,6 +505,8 @@ TEST_F(CollectionAllFieldsTest, StringSingularAllValues) {ASSERT_EQ(1, results["hits"].size());ASSERT_EQ("FIRST", results["hits"][0]["document"]["title"].get<std::string>());ASSERT_EQ("123", results["hits"][0]["document"]["int_values"].get<std::string>());++ collectionManager.drop_collection("coll1");}TEST_F(CollectionAllFieldsTest, UpdateOfDocumentsInAutoMode) {diff --git a/test/collection_test.cpp b/test/collection_test.cppindex fb87efab..dab4a2f9 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -75,7 +75,7 @@ protected:std::vector<std::string> strs;for(size_t i = 0 ; i < num_words ; i++ ) {- int word_index = rand() % 100;+ int word_index = rand() % words.size();strs.push_back(words[word_index]);}return StringUtils::join(strs, " ");@@ -1182,6 +1182,50 @@ TEST_F(CollectionTest, ImportDocumentsUpsert) {ASSERT_EQ(409, import_results[1]["code"].get<size_t>());}+TEST_F(CollectionTest, DISABLED_CrashTroubleshooting) {+ Collection *coll1;+ std::vector<field> fields = {+ field("title", field_types::STRING_ARRAY, false, true),+ field("points", field_types::INT32, false)+ };++ coll1 = collectionManager.get_collection("coll1").get();+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();+ }++ std::ifstream create_file("/tmp/create.jsonl");+ std::string json_line;+ std::vector<std::string> create_records;++ while (std::getline(create_file, json_line)) {+ create_records.push_back(json_line);+ }++ create_file.close();++ nlohmann::json document;+ auto import_response = coll1->add_many(create_records, document, CREATE);++ ASSERT_TRUE(import_response["success"].get<bool>());+ ASSERT_EQ(1000, import_response["num_imported"].get<int>());++ // now try to upsert++ std::ifstream upsert_file("/tmp/upsert.jsonl");+ std::vector<std::string> upsert_records;++ while (std::getline(upsert_file, json_line)) {+ upsert_records.push_back(json_line);+ }++ upsert_file.close();++ import_response = coll1->add_many(upsert_records, document, UPSERT);++ ASSERT_TRUE(import_response["success"].get<bool>());+ ASSERT_EQ(1000, import_response["num_imported"].get<int>());+}TEST_F(CollectionTest, ImportDocumentsUpsertOptional) {Collection *coll1;