Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Posting list integration.

Kishore Nallan929b6796ccf4
diff --git a/TODO.md b/TODO.md
index fdbf335c..0a380d7a 100644
--- a/TODO.md
+++ b/TODO.md
@@ -3,6 +3,9 @@
- Writes on dedicated threads
- Single field aggregation short circuit
- Test for group + multiple fields
+- Intersect with single posting list
+- Test for erase dropping elements below compressed list threshold
+- Test for array token positions
**Search index**
diff --git a/include/art.h b/include/art.h
index 89f55da9..976a47c5 100644
--- a/include/art.h
+++ b/include/art.h
@@ -99,7 +99,7 @@ typedef struct {
typedef struct {
uint32_t key_len;
int64_t max_score;
- art_values* values;
+ void* values;
unsigned char key[];
} art_leaf;
@@ -118,8 +118,7 @@ typedef struct {
typedef struct {
int32_t score;
uint32_t id;
- uint32_t offsets_len;
- uint32_t* offsets;
+ std::vector<uint32_t> offsets;
} art_document;
enum token_ordering {
@@ -186,7 +185,7 @@ inline uint64_t art_size(art_tree *t) {
* @return NULL if the item was newly inserted, otherwise
* the old value pointer is returned.
*/
-void* art_insert(art_tree *t, const unsigned char *key, int key_len, art_document* document, uint32_t num_hits);
+void* art_insert(art_tree *t, const unsigned char *key, int key_len, art_document* document);
/**
* Deletes a value from the ART tree
diff --git a/include/index.h b/include/index.h
index fcd1057c..c123a92f 100644
--- a/include/index.h
+++ b/include/index.h
@@ -20,6 +20,7 @@
#include "num_tree.h"
#include "magic_enum.hpp"
#include "match_score.h"
+#include "posting_list.h"
struct token_t {
size_t position;
@@ -309,8 +310,10 @@ public:
void score_results(const std::vector<sort_by> &sort_fields, const uint16_t &query_index, const uint8_t &field_id,
const uint32_t total_cost, Topster *topster, const std::vector<art_leaf *> &query_suggestion,
- spp::sparse_hash_set<uint64_t> &groups_processed, const uint32_t *result_ids,
- const size_t result_size, const size_t group_limit,
+ spp::sparse_hash_set<uint64_t> &groups_processed,
+ const std::vector<std::unordered_map<size_t, std::vector<token_positions_t>>>& array_token_positions_vec,
+ const uint32_t* result_ids, size_t result_ids_size,
+ const size_t group_limit,
const std::vector<std::string> &group_by_fields, uint32_t token_bits,
const std::vector<token_t> &query_tokens, bool prioritize_exact_match) const;
@@ -326,10 +329,6 @@ public:
uint64_t get_distinct_id(const std::vector<std::string>& group_by_fields, const uint32_t seq_id) const;
- void eq_str_filter_plain(const uint32_t *strt_ids, size_t strt_ids_size,
- const std::vector<art_leaf *> &query_suggestion,
- uint32_t *exact_strt_ids, size_t& exact_strt_size) const;
-
void scrub_reindex_doc(nlohmann::json& update_doc, nlohmann::json& del_doc, nlohmann::json& old_doc);
static void tokenize_string_field(const nlohmann::json& document,
@@ -374,16 +373,6 @@ public:
const std::map<std::string, field> & facet_schema,
const std::string& fallback_field_type);
- static void populate_token_positions(const std::vector<art_leaf *> &query_suggestion,
- const std::vector<uint32_t*>& leaf_to_indices,
- const size_t result_index,
- std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_positions);
-
- /*static bool is_exact_token_match(const art_leaf* leaf,
- const std::vector<uint32_t*>& leaf_to_indices,
- const size_t result_index,
- std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_positions);
-*/
static bool is_point_in_polygon(const Geofence& poly, const GeoCoord& point);
static double transform_for_180th_meridian(Geofence& poly);
diff --git a/src/art.cpp b/src/art.cpp
index 0114fcd4..0ff09564 100644
--- a/src/art.cpp
+++ b/src/art.cpp
@@ -12,6 +12,7 @@
#include <limits>
#include <queue>
#include <stdint.h>
+#include <posting.h>
#include "art.h"
#include "logger.h"
@@ -39,10 +40,8 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *
void art_int_fuzzy_recurse(art_node *n, int depth, const unsigned char* int_str, int int_str_len,
NUM_COMPARATOR comparator, std::vector<const art_leaf *> &results);
-static void insert_and_shift_offset_index(sorted_array& offset_index, const uint32_t index, const uint32_t num_offsets);
-
bool compare_art_leaf_frequency(const art_leaf *a, const art_leaf *b) {
- return a->values->ids.getLength() > b->values->ids.getLength();
+ return posting_t::num_ids(a->values) > posting_t::num_ids(b->values);
}
bool compare_art_leaf_score(const art_leaf *a, const art_leaf *b) {
@@ -54,12 +53,12 @@ bool compare_art_node_frequency(const art_node *a, const art_node *b) {
if(IS_LEAF(a)) {
art_leaf* al = (art_leaf *) LEAF_RAW(a);
- a_value = al->values->ids.getLength();
+ a_value = posting_t::num_ids(al->values);
}
if(IS_LEAF(b)) {
art_leaf* bl = (art_leaf *) LEAF_RAW(b);
- b_value = bl->values->ids.getLength();
+ b_value = posting_t::num_ids(bl->values);
}
return a_value > b_value;
@@ -138,7 +137,7 @@ static void destroy_node(art_node *n) {
// Special case leafs
if (IS_LEAF(n)) {
art_leaf *leaf = (art_leaf *) LEAF_RAW(n);
- delete leaf->values;
+ posting_t::destroy_list(leaf->values);
free(leaf);
return;
}
@@ -415,47 +414,20 @@ art_leaf* art_maximum(art_tree *t) {
static void add_document_to_leaf(const art_document *document, art_leaf *leaf) {
leaf->max_score = MAX(leaf->max_score, document->score);
- size_t inserted_index = leaf->values->ids.append(document->id);
-
- if(inserted_index == leaf->values->ids.getLength()-1) {
- // treat as appends
- uint32_t curr_index = leaf->values->offsets.getLength();
- leaf->values->offset_index.append(curr_index);
- for(uint32_t i=0; i<document->offsets_len; i++) {
- leaf->values->offsets.append(document->offsets[i]);
- }
- } else {
- uint32_t existing_offset_index = leaf->values->offset_index.at(inserted_index);
- insert_and_shift_offset_index(leaf->values->offset_index, inserted_index, document->offsets_len);
- leaf->values->offsets.insert(existing_offset_index, document->offsets, document->offsets_len);
- }
-}
-
-void insert_and_shift_offset_index(sorted_array& offset_index, const uint32_t index, const uint32_t num_offsets) {
- uint32_t existing_offset_index = offset_index.at(index);
- uint32_t length = offset_index.getLength();
- uint32_t new_length = length + 1;
- uint32_t *curr_array = offset_index.uncompress(new_length);
-
- memmove(&curr_array[index+1], &curr_array[index], sizeof(uint32_t)*(length - index));
- curr_array[index] = existing_offset_index;
-
- uint32_t curr_index = index + 1;
- while(curr_index < new_length) {
- curr_array[curr_index] += num_offsets;
- curr_index++;
- }
-
- offset_index.load(curr_array, new_length);
-
- delete [] curr_array;
+ posting_t::upsert(leaf->values, document->id, document->offsets);
}
static art_leaf* make_leaf(const unsigned char *key, uint32_t key_len, art_document *document) {
art_leaf *l = (art_leaf *) malloc(sizeof(art_leaf) + key_len);
- l->values = new art_values;
- l->max_score = 0;
l->key_len = key_len;
+ l->max_score = 0;
+
+ uint32_t ids[1] = {document->id};
+ uint32_t offset_index[1] = {0};
+ compact_posting_list_t* list = compact_posting_list_t::create(1, ids, offset_index, document->offsets.size(),
+ &document->offsets[0]);
+ l->values = SET_COMPACT_POSTING(list);
+
memcpy(l->key, key, key_len);
add_document_to_leaf(document, l);
return l;
@@ -622,7 +594,7 @@ static int prefix_mismatch(const art_node *n, const unsigned char *key, int key_
return idx;
}
-static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *key, uint32_t key_len, art_document *document, uint32_t num_hits, int depth, int *old) {
+static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *key, uint32_t key_len, art_document *document, int depth, int *old) {
// If we are at a NULL node, inject a leaf
if (!n) {
*ref = (art_node*)SET_LEAF(make_leaf(key, key_len, document));
@@ -636,14 +608,8 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *
// Check if we are updating an existing value
if (!leaf_matches(l, key, key_len, depth)) {
*old = 1;
- art_values *ret_val = l->values;
-
- // updates are not supported
- if(!l->values->ids.contains(document->id)) {
- add_document_to_leaf(document, l);
- }
-
- return ret_val;
+ add_document_to_leaf(document, l);
+ return l->values;
}
// New value, we must split the leaf into a node4
@@ -705,7 +671,7 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *
// Find a child to recurse to
art_node **child = find_child(n, key[depth]);
if (child) {
- return recursive_insert(*child, child, key, key_len, document, num_hits, depth + 1, old);
+ return recursive_insert(*child, child, key, key_len, document, depth + 1, old);
}
// No child, node goes within us
@@ -723,10 +689,10 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *
* @return NULL if the item was newly inserted, otherwise
* the old value pointer is returned.
*/
-void* art_insert(art_tree *t, const unsigned char *key, int key_len, art_document* document, uint32_t num_hits) {
+void* art_insert(art_tree *t, const unsigned char *key, int key_len, art_document* document) {
int old_val = 0;
- void *old = recursive_insert(t->root, &t->root, key, key_len, document, num_hits, 0, &old_val);
+ void *old = recursive_insert(t->root, &t->root, key, key_len, document, 0, &old_val);
if (!old_val) t->size++;
return old;
}
@@ -951,8 +917,8 @@ int art_topk_iter(const art_node *root, token_ordering token_order, size_t max_r
results.push_back(l);
} else {
// we will push leaf only if filter matches with leaf IDs
- size_t found_len = l->values->ids.numFoundOf(filter_ids, filter_ids_length);
- if(found_len != 0) {
+ bool found_atleast_one = posting_t::contains_atleast_one(l->values, filter_ids, filter_ids_length);
+ if(found_atleast_one) {
results.push_back(l);
}
}
diff --git a/src/collection.cpp b/src/collection.cpp
index 4414b39d..84b7bc70 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -15,6 +15,7 @@
#include <h3api.h>
#include <regex>
#include <list>
+#include <posting.h>
#include "topster.h"
#include "logger.h"
@@ -1478,11 +1479,10 @@ void Collection::highlight_result(const field &search_field,
return ;
}
- std::vector<uint32_t*> leaf_to_indices;
std::vector<art_leaf*> query_suggestion;
std::set<std::string> query_suggestion_tokens;
- for (const art_leaf *token_leaf : searched_queries[field_order_kv->query_index]) {
+ for (const art_leaf* token_leaf : searched_queries[field_order_kv->query_index]) {
// Must search for the token string fresh on that field for the given document since `token_leaf`
// is from the best matched field and need not be present in other fields of a document.
Index* index = indices[field_order_kv->key % num_memory_shards];
@@ -1494,11 +1494,6 @@ void Collection::highlight_result(const field &search_field,
query_suggestion.push_back(actual_leaf);
std::string token(reinterpret_cast<char*>(actual_leaf->key), actual_leaf->key_len-1);
query_suggestion_tokens.insert(token);
- std::vector<uint16_t> positions;
- uint32_t doc_index = actual_leaf->values->ids.indexOf(field_order_kv->key);
- auto doc_indices = new uint32_t[1];
- doc_indices[0] = doc_index;
- leaf_to_indices.push_back(doc_indices);
}
}
@@ -1514,30 +1509,31 @@ void Collection::highlight_result(const field &search_field,
reinterpret_cast<const unsigned char *>(q_token.c_str()),
q_token.size() + 1);
if(actual_leaf != nullptr) {
- std::vector<uint16_t> positions;
- uint32_t doc_index = actual_leaf->values->ids.indexOf(field_order_kv->key);
- if(doc_index != actual_leaf->values->ids.getLength()) {
- auto doc_indices = new uint32_t[1];
- doc_indices[0] = doc_index;
- leaf_to_indices.push_back(doc_indices);
- query_suggestion.push_back(actual_leaf);
- }
+ query_suggestion.push_back(actual_leaf);
}
}
}
if(query_suggestion.empty()) {
// none of the tokens from the query were found on this field
- free_leaf_indices(leaf_to_indices);
return ;
}
//LOG(INFO) << "Document ID: " << document["id"];
- // positions in the field of each token in the query
- std::unordered_map<size_t, std::vector<token_positions_t>> array_token_positions;
- Index::populate_token_positions(query_suggestion, leaf_to_indices, 0, array_token_positions);
+ std::vector<void*> posting_lists;
+ for(art_leaf* leaf: query_suggestion) {
+ posting_lists.push_back(leaf->values);
+ }
+
+ std::vector<std::unordered_map<size_t, std::vector<token_positions_t>>> array_token_positions_vec;
+ posting_t::get_array_token_positions(field_order_kv->key, posting_lists, array_token_positions_vec);
+ if(array_token_positions_vec.empty()) {
+ return;
+ }
+
+ std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_positions = array_token_positions_vec[0];
std::vector<match_index_t> match_indices;
for(const auto& kv: array_token_positions) {
@@ -1559,7 +1555,6 @@ void Collection::highlight_result(const field &search_field,
if(match_indices.empty()) {
// none of the tokens from the query were found on this field
- free_leaf_indices(leaf_to_indices);
return ;
}
@@ -1707,8 +1702,6 @@ void Collection::highlight_result(const field &search_field,
highlight.field = search_field.name;
highlight.match_score = match_indices[0].match_score;
-
- free_leaf_indices(leaf_to_indices);
}
void Collection::free_leaf_indices(std::vector<uint32_t*>& leaf_to_indices) const {
diff --git a/src/index.cpp b/src/index.cpp
index 93f157a8..5d6295ba 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -17,6 +17,7 @@
#include <s2/s2earth.h>
#include <s2/s2loop.h>
#include <s2/s2builder.h>
+#include <posting.h>
#include "logger.h"
Index::Index(const std::string name, const std::unordered_map<std::string, field> & search_schema,
@@ -506,29 +507,15 @@ void Index::insert_doc(const int64_t score, art_tree *t, uint32_t seq_id,
art_document art_doc;
art_doc.id = seq_id;
art_doc.score = score;
- art_doc.offsets_len = (uint32_t) kv.second.size();
- art_doc.offsets = new uint32_t[kv.second.size()];
+ art_doc.offsets = kv.second;
uint32_t num_hits = 0;
const unsigned char *key = (const unsigned char *) kv.first.c_str();
int key_len = (int) kv.first.length() + 1; // for the terminating \0 char
- art_leaf* leaf = (art_leaf *) art_search(t, key, key_len);
- if(leaf != NULL) {
- num_hits = leaf->values->ids.getLength();
- }
-
- num_hits += 1;
-
- for(size_t i=0; i<kv.second.size(); i++) {
- art_doc.offsets[i] = kv.second[i];
- }
-
//LOG(INFO) << "key: " << key << ", art_doc.id: " << art_doc.id;
- art_insert(t, key, key_len, &art_doc, num_hits);
- delete [] art_doc.offsets;
- art_doc.offsets = nullptr;
+ art_insert(t, key, key_len, &art_doc);
}
}
@@ -929,95 +916,106 @@ void Index::search_candidates(const uint8_t & field_id,
}*/
// initialize results with the starting element (for further intersection)
- size_t result_size = query_suggestion[0]->values->ids.getLength();
+ size_t result_size = posting_t::num_ids(query_suggestion[0]->values);
if(result_size == 0) {
continue;
}
- uint32_t* result_ids = query_suggestion[0]->values->ids.uncompress();
+ // Prepare excluded document IDs that we can later remove from the result set
+ uint32_t* excluded_result_ids = nullptr;
+ size_t excluded_result_ids_size = ArrayUtils::or_scalar(exclude_token_ids, exclude_token_ids_size,
+ &curated_ids[0], curated_ids.size(), &excluded_result_ids);
- // intersect the document ids for each token to find docs that contain all the tokens (stored in `result_ids`)
- for(size_t i=1; i < query_suggestion.size(); i++) {
- uint32_t* out = nullptr;
- uint32_t* ids = query_suggestion[i]->values->ids.uncompress();
- result_size = ArrayUtils::and_scalar(ids, query_suggestion[i]->values->ids.getLength(), result_ids, result_size, &out);
- delete[] ids;
- delete[] result_ids;
- result_ids = out;
- }
+ std::vector<void*> posting_lists;
- if(result_size == 0) {
- delete[] result_ids;
- continue;
+ for(auto& query_leaf : query_suggestion) {
+ posting_lists.push_back(query_leaf->values);
}
- // Exclude document IDs associated with excluded tokens from the result set
- if(exclude_token_ids_size != 0) {
- uint32_t *excluded_result_ids = nullptr;
- result_size = ArrayUtils::exclude_scalar(result_ids, result_size, exclude_token_ids, exclude_token_ids_size,
- &excluded_result_ids);
- delete[] result_ids;
- result_ids = excluded_result_ids;
- }
+ std::vector<posting_list_t::iterator_t> its;
+ posting_list_t::result_iter_state_t iter_state;
+ std::vector<uint32_t> result_id_vec;
- if(!curated_ids.empty()) {
- uint32_t *excluded_result_ids = nullptr;
- result_size = ArrayUtils::exclude_scalar(result_ids, result_size, &curated_ids[0],
- curated_ids.size(), &excluded_result_ids);
+ size_t excluded_result_ids_index = 0;
+ size_t filter_ids_index = 0;
- delete [] result_ids;
- result_ids = excluded_result_ids;
- }
+ posting_t::block_intersector_t intersector(posting_lists, 1000, iter_state);
+ bool has_more = true;
- //LOG(INFO) << "n: " << n;
- /*std::stringstream log_query;
- for(size_t i=0; i < query_suggestion.size(); i++) {
- log_query << query_suggestion[i]->key << " ";
- }*/
+ while(has_more) {
+ has_more = intersector.intersect();
+ posting_list_t::result_iter_state_t updated_iter_state;
- if(filter_ids != nullptr) {
- // intersect once again with filter ids
- uint32_t* filtered_result_ids = nullptr;
- size_t filtered_results_size = ArrayUtils::and_scalar(filter_ids, filter_ids_length, result_ids,
- result_size, &filtered_result_ids);
+ for(size_t i = 0; i < iter_state.ids.size(); i++) {
+ uint32_t id = iter_state.ids[i];
- uint32_t* new_all_result_ids = nullptr;
- all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, filtered_result_ids,
- filtered_results_size, &new_all_result_ids);
- delete [] *all_result_ids;
- *all_result_ids = new_all_result_ids;
+ // decide if this result id should be excluded
+ if(excluded_result_ids_size != 0) {
+ while(excluded_result_ids_index < excluded_result_ids_size &&
+ excluded_result_ids[excluded_result_ids_index] < id) {
+ excluded_result_ids_index++;
+ }
- // go through each matching document id and calculate match score
- score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster,
- query_suggestion,
- groups_processed, filtered_result_ids, filtered_results_size,
- group_limit, group_by_fields, token_bits, query_tokens, prioritize_exact_match);
+ if(excluded_result_ids_index < excluded_result_ids_size &&
+ id == excluded_result_ids[excluded_result_ids_index]) {
+ excluded_result_ids_index++;
+ continue;
+ }
+ }
- field_num_results += filtered_results_size;
+ bool id_found_in_filter = true;
- delete[] filtered_result_ids;
- delete[] result_ids;
- } else {
- uint32_t* new_all_result_ids = nullptr;
- all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, result_ids,
- result_size, &new_all_result_ids);
- delete [] *all_result_ids;
- *all_result_ids = new_all_result_ids;
+ // decide if this result be matched with filter results
+ if(filter_ids_length != 0) {
+ id_found_in_filter = false;
- /*if(result_size != 0) {
- LOG(INFO) << size_t(field_id) << " - " << log_query.str() << ", result_size: " << result_size;
- }*/
+ // e.g. [1, 3] vs [2, 3]
- score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster,
- query_suggestion,
- groups_processed, result_ids, result_size, group_limit, group_by_fields, token_bits,
- query_tokens, prioritize_exact_match);
+ while(filter_ids_index < filter_ids_length && filter_ids[filter_ids_index] < id) {
+ filter_ids_index++;
+ }
- field_num_results += result_size;
+ if(filter_ids_index < filter_ids_length && filter_ids[filter_ids_index] == id) {
+ filter_ids_index++;
+ id_found_in_filter = true;
+ }
+ }
+
+ if(id_found_in_filter) {
+ result_id_vec.push_back(id);
- delete[] result_ids;
+ updated_iter_state.ids.push_back(id);
+ updated_iter_state.blocks.push_back(iter_state.blocks[i]);
+ updated_iter_state.indices.push_back(iter_state.indices[i]);
+ }
+ }
+
+ std::vector<std::unordered_map<size_t, std::vector<token_positions_t>>> array_token_positions_vec;
+ posting_list_t::get_offsets(updated_iter_state, array_token_positions_vec);
+
+ score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster,
+ query_suggestion, groups_processed, array_token_positions_vec,
+ &updated_iter_state.ids[0], updated_iter_state.ids.size(),
+ group_limit, group_by_fields, token_bits, query_tokens, prioritize_exact_match);
+ }
+
+ if(result_id_vec.empty()) {
+ continue;
}
+ uint32_t* result_ids = &result_id_vec[0];
+ result_size = result_id_vec.size();
+
+ field_num_results += result_id_vec.size();
+
+ uint32_t* new_all_result_ids = nullptr;
+ all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, result_ids,
+ result_size, &new_all_result_ids);
+ delete [] *all_result_ids;
+ *all_result_ids = new_all_result_ids;
+
+ delete [] excluded_result_ids;
+
searched_queries.push_back(actual_query_suggestion);
//LOG(INFO) << "field_num_results: " << field_num_results << ", typo_tokens_threshold: " << typo_tokens_threshold;
@@ -1179,7 +1177,7 @@ uint32_t Index::do_filtering(uint32_t** filter_ids_out, const std::vector<filter
uint32_t* strt_ids = nullptr;
size_t strt_ids_size = 0;
- std::vector<art_leaf *> query_suggestion;
+ std::vector<void*> posting_lists;
// there could be multiple tokens in a filter value, which we have to treat as ANDs
// e.g. country: South Africa
@@ -1199,27 +1197,19 @@ uint32_t Index::do_filtering(uint32_t** filter_ids_out, const std::vector<filter
continue;
}
- query_suggestion.push_back(leaf);
+ posting_lists.push_back(leaf->values);
}
- if(query_suggestion.size() != str_tokens.size()) {
+ if(posting_lists.size() != str_tokens.size()) {
continue;
}
- for(const auto& leaf: query_suggestion) {
- if(strt_ids == nullptr) {
- strt_ids = leaf->values->ids.uncompress();
- strt_ids_size = leaf->values->ids.getLength();
- } else {
- // do AND for an exact match
- uint32_t* out = nullptr;
- uint32_t* leaf_ids = leaf->values->ids.uncompress();
- strt_ids_size = ArrayUtils::and_scalar(strt_ids, strt_ids_size, leaf_ids,
- leaf->values->ids.getLength(), &out);
- delete[] leaf_ids;
- delete[] strt_ids;
- strt_ids = out;
- }
+ std::vector<uint32_t> result_id_vec;
+ posting_t::intersect(posting_lists, result_id_vec);
+ if(!result_id_vec.empty()) {
+ strt_ids = new uint32_t [result_id_vec.size()];
+ std::copy(result_id_vec.begin(), result_id_vec.end(), strt_ids);
+ strt_ids_size = result_id_vec.size();
}
if((a_filter.comparators[0] == EQUALS || a_filter.comparators[0] == NOT_EQUALS) && f.is_facet()) {
@@ -1234,7 +1224,7 @@ uint32_t Index::do_filtering(uint32_t** filter_ids_out, const std::vector<filter
bool found_filter = false;
if(!f.is_array()) {
- found_filter = (query_suggestion.size() == fvalues.length);
+ found_filter = (posting_lists.size() == fvalues.length);
} else {
uint64_t filter_hash = 1;
@@ -1337,59 +1327,6 @@ uint32_t Index::do_filtering(uint32_t** filter_ids_out, const std::vector<filter
return filter_ids_length;
}
-void Index::eq_str_filter_plain(const uint32_t *strt_ids, size_t strt_ids_size,
- const std::vector<art_leaf *>& query_suggestion, uint32_t *exact_strt_ids,
- size_t& exact_strt_size) const {
- std::vector<uint32_t*> leaf_to_indices;
- for (art_leaf *token_leaf: query_suggestion) {
- if(token_leaf == nullptr) {
- leaf_to_indices.push_back(nullptr);
- continue;
- }
-
- uint32_t *indices = new uint32_t[strt_ids_size];
- token_leaf->values->ids.indexOf(strt_ids, strt_ids_size, indices);
- leaf_to_indices.push_back(indices);
- }
-
- // e.g. First In First Out => hash([0, 1, 0, 2])
- spp::sparse_hash_map<size_t, uint32_t> leaf_to_id;
- size_t next_id = 1;
- size_t filter_hash = 1;
-
- for(size_t leaf_index=0; leaf_index<query_suggestion.size(); leaf_index++) {
- if(leaf_to_id.count(leaf_index) == 0) {
- leaf_to_id.emplace(leaf_index, next_id++);
- }
-
- uint32_t leaf_id = leaf_to_id[leaf_index];
- filter_hash *= (1779033703 + 2*leaf_id*(leaf_index+1));
- }
-
- for(size_t strt_ids_index = 0; strt_ids_index < strt_ids_size; strt_ids_index++) {
- std::unordered_map<size_t, std::vector<token_positions_t>> array_token_positions;
- populate_token_positions(query_suggestion, leaf_to_indices, strt_ids_index, array_token_positions);
- // iterate array_token_positions and compute hash
-
- for(const auto& kv: array_token_positions) {
- const std::vector<token_positions_t>& token_positions = kv.second;
- size_t this_hash = 1;
-
- for(size_t token_index = 0; token_index < token_positions.size(); token_index++) {
- auto& positions = token_positions[token_index].positions;
- for(auto pos: positions) {
- this_hash *= (1779033703 + 2*(token_index+1)*(pos+1));
- }
- }
-
- if(this_hash == filter_hash) {
- exact_strt_ids[exact_strt_size++] = strt_ids[strt_ids_index];
- break;
- }
- }
- }
-}
-
void Index::run_search(search_args* search_params) {
search(search_params->field_query_tokens,
search_params->search_fields,
@@ -1535,22 +1472,30 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,
// find documents that contain the excluded tokens to exclude them from results later
for(size_t i = 0; i < num_search_fields; i++) {
const std::string & field_name = search_fields[i].name;
+
+ std::vector<void*> posting_lists;
+
for(const std::string& exclude_token: field_query_tokens[i].q_exclude_tokens) {
art_leaf* leaf = (art_leaf *) art_search(search_index.at(field_name),
(const unsigned char *) exclude_token.c_str(),
exclude_token.size() + 1);
if(leaf) {
- uint32_t *ids = leaf->values->ids.uncompress();
- uint32_t *exclude_token_ids_merged = nullptr;
- exclude_token_ids_size = ArrayUtils::or_scalar(exclude_token_ids, exclude_token_ids_size, ids,
- leaf->values->ids.getLength(),
- &exclude_token_ids_merged);
- delete[] ids;
- delete[] exclude_token_ids;
- exclude_token_ids = exclude_token_ids_merged;
+ posting_lists.push_back(leaf->values);
}
}
+
+ std::vector<uint32_t> exclude_token_id_vec;
+ if(!posting_lists.empty()) {
+ posting_t::merge(posting_lists, exclude_token_id_vec);
+ }
+
+ uint32_t *exclude_token_ids_merged = nullptr;
+ exclude_token_ids_size = ArrayUtils::or_scalar(exclude_token_ids, exclude_token_ids_size,
+ &exclude_token_id_vec[0], exclude_token_id_vec.size(),
+ &exclude_token_ids_merged);
+ delete[] exclude_token_ids;
+ exclude_token_ids = exclude_token_ids_merged;
}
std::vector<Topster*> ftopsters;
@@ -1599,7 +1544,7 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,
uint32_t token_bits = 255;
score_results(sort_fields_std, (uint16_t) searched_queries.size(), field_id, 0, topster, {},
- groups_processed, filter_ids, filter_ids_length, group_limit, group_by_fields, token_bits, {},
+ groups_processed, {}, filter_ids, filter_ids_length, group_limit, group_by_fields, token_bits, {},
prioritize_exact_match);
collate_included_ids(field_query_tokens[0].q_include_tokens, field, field_id, included_ids_map, curated_topster, searched_queries);
@@ -1752,8 +1697,7 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,
continue;
}
- uint32_t doc_index = leaves[0]->values->ids.indexOf(seq_id);
- if (doc_index == leaves[0]->values->ids.getLength()) {
+ if(!posting_t::contains(leaves[0]->values, seq_id)) {
continue;
}
@@ -2016,8 +1960,8 @@ void Index::log_leaves(const int cost, const std::string &token, const std::vect
for(size_t i=0; i < leaves.size(); i++) {
std::string key((char*)leaves[i]->key, leaves[i]->key_len);
- LOG(INFO) << key << " - " << leaves[i]->values->ids.getLength();
- LOG(INFO) << "frequency: " << leaves[i]->values->ids.getLength() << ", max_score: " << leaves[i]->max_score;
+ LOG(INFO) << key << " - " << posting_t::num_ids(leaves[i]->values);
+ LOG(INFO) << "frequency: " << posting_t::num_ids(leaves[i]->values) << ", max_score: " << leaves[i]->max_score;
/*for(auto j=0; j<leaves[i]->values->ids.getLength(); j++) {
LOG(INFO) << "id: " << leaves[i]->values->ids.at(j);
}*/
@@ -2028,7 +1972,8 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
const uint8_t & field_id, const uint32_t total_cost, Topster* topster,
const std::vector<art_leaf *> &query_suggestion,
spp::sparse_hash_set<uint64_t>& groups_processed,
- const uint32_t *result_ids, const size_t result_size,
+ const std::vector<std::unordered_map<size_t, std::vector<token_positions_t>>>& array_token_positions_vec,
+ const uint32_t* result_ids, size_t result_ids_size,
const size_t group_limit, const std::vector<std::string>& group_by_fields,
uint32_t token_bits,
const std::vector<token_t>& query_tokens,
@@ -2060,7 +2005,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
S2LatLng reference_lat_lng;
GeoPoint::unpack_lat_lng(sort_fields[i].geopoint, reference_lat_lng);
- for (size_t rindex = 0; rindex < result_size; rindex++) {
+ for (size_t rindex = 0; rindex < result_ids_size; rindex++) {
const uint32_t seq_id = result_ids[rindex];
auto it = geopoints->find(seq_id);
int64_t dist = INT32_MAX;
@@ -2085,13 +2030,6 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
}
}
- std::vector<uint32_t *> leaf_to_indices;
- for (art_leaf *token_leaf: query_suggestion) {
- uint32_t *indices = new uint32_t[result_size];
- token_leaf->values->ids.indexOf(result_ids, result_size, indices);
- leaf_to_indices.push_back(indices);
- }
-
Match single_token_match = Match(1, 0);
const uint64_t single_token_match_score = single_token_match.get_match_score(total_cost);
@@ -2103,16 +2041,16 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
(query_suggestion.size() == query_tokens.size()) &&
((std::string((const char*)query_suggestion[0]->key, query_suggestion[0]->key_len-1) != query_tokens[0].value));
- for (size_t i = 0; i < result_size; i++) {
+ for (size_t i = 0; i < result_ids_size; i++) {
const uint32_t seq_id = result_ids[i];
uint64_t match_score = 0;
- if (use_single_token_score) {
+ if (use_single_token_score || array_token_positions_vec.empty()) {
match_score = single_token_match_score;
} else {
- std::unordered_map<size_t, std::vector<token_positions_t>> array_token_positions;
- populate_token_positions(query_suggestion, leaf_to_indices, i, array_token_positions);
+ const std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_positions =
+ array_token_positions_vec[i];
for (const auto& kv: array_token_positions) {
const std::vector<token_positions_t>& token_positions = kv.second;
@@ -2201,10 +2139,6 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
//long long int timeNanos = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();
//LOG(INFO) << "Time taken for results iteration: " << timeNanos << "ms";
-
- for(uint32_t* leaf_indices: leaf_to_indices) {
- delete [] leaf_indices;
- }
}
// pre-filter group_by_fields such that we can avoid the find() check
@@ -2236,103 +2170,6 @@ uint64_t Index::get_distinct_id(const std::vector<std::string>& group_by_fields,
return distinct_id;
}
-void Index::populate_token_positions(const std::vector<art_leaf *>& query_suggestion,
- const std::vector<uint32_t*>& leaf_to_indices,
- const size_t result_index,
- std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_positions) {
- if(query_suggestion.empty()) {
- return ;
- }
-
- // array_token_positions:
- // for every element in a potential array, for every token in query suggestion, get the positions
-
- for(size_t i = 0; i < query_suggestion.size(); i++) {
- const art_leaf* token_leaf = query_suggestion[i];
- uint32_t doc_index = leaf_to_indices[i][result_index];
- /*LOG(INFO) << "doc_id: " << token_leaf->values->ids.at(doc_index) << ", token_leaf->values->ids.getLength(): "
- << token_leaf->values->ids.getLength();*/
-
- // it's possible for a query token to not appear in a resulting document
- if(doc_index == token_leaf->values->ids.getLength()) {
- continue;
- }
-
- //LOG(INFO) << "key: " << token_leaf->key;
-
- // Plain string format:
- // offset1, offset2, ... , 0 (if token is the last offset for the document)
-
- // Array string format:
- // offset1, ... , offsetn, offsetn, array_index, 0 (if token is the last offset for the document)
- // (last offset is repeated to indicate end of offsets for a given array index)
-
- /*uint32_t* offsets = token_leaf->values->offsets.uncompress();
- for(size_t ii=0; ii < token_leaf->values->offsets.getLength(); ii++) {
- LOG(INFO) << "offset: " << offsets[ii];
- }
-
- uint32_t* offset_indices = token_leaf->values->offset_index.uncompress();
- for(size_t ii=0; ii < token_leaf->values->offset_index.getLength(); ii++) {
- LOG(INFO) << "offset index: " << offset_indices[ii];
- }
-
- LOG(INFO) << "token_leaf->values->offsets.getLength(): " << token_leaf->values->offsets.getLength();*/
-
- uint32_t start_offset = token_leaf->values->offset_index.at(doc_index);
- uint32_t end_offset = (doc_index == token_leaf->values->ids.getLength() - 1) ?
- token_leaf->values->offsets.getLength() :
- token_leaf->values->offset_index.at(doc_index+1);
-
- std::vector<uint16_t> positions;
- int prev_pos = -1;
- bool is_last_token = false;
-
- while(start_offset < end_offset) {
- int pos = token_leaf->values->offsets.at(start_offset);
- start_offset++;
-
- if(pos == 0) {
- // indicates that token is the last token on the doc
- is_last_token = true;
- start_offset++;
- continue;
- }
-
- if(pos == prev_pos) { // indicates end of array index
- if(!positions.empty()) {
- size_t array_index = (size_t) token_leaf->values->offsets.at(start_offset);
- is_last_token = false;
-
- if(start_offset+1 < end_offset) {
- size_t next_offset = (size_t) token_leaf->values->offsets.at(start_offset + 1);
- if(next_offset == 0) {
- // indicates that token is the last token on the doc
- is_last_token = true;
- start_offset++;
- }
- }
-
- array_token_positions[array_index].push_back(token_positions_t{is_last_token, positions});
- positions.clear();
- }
-
- start_offset++; // skip current value which is the array index or flag for last index
- prev_pos = -1;
- continue;
- }
-
- prev_pos = pos;
- positions.push_back((uint16_t)pos - 1);
- }
-
- if(!positions.empty()) {
- // for plain string fields
- array_token_positions[0].push_back(token_positions_t{is_last_token, positions});
- }
- }
-}
-
inline uint32_t Index::next_suggestion(const std::vector<token_candidates> &token_candidates_vec,
long long int n,
std::vector<art_leaf *>& actual_query_suggestion,
@@ -2359,7 +2196,7 @@ inline uint32_t Index::next_suggestion(const std::vector<token_candidates> &toke
// Sort ascending based on matched documents for each token for faster intersection.
// However, this causes the token order to deviate from original query's order.
sort(query_suggestion.begin(), query_suggestion.end(), [](const art_leaf* left, const art_leaf* right) {
- return left->values->ids.getLength() < right->values->ids.getLength();
+ return posting_t::num_ids(left->values) < posting_t::num_ids(right->values);
});
return total_cost;
@@ -2419,39 +2256,17 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & doc
std::vector<std::string> tokens;
tokenize_string_field(document, search_field, tokens, search_field.locale);
- for(auto & token: tokens) {
+ for(size_t i = 0; i < tokens.size(); i++) {
+ const auto& token = tokens[i];
const unsigned char *key = (const unsigned char *) token.c_str();
int key_len = (int) (token.length() + 1);
art_leaf* leaf = (art_leaf *) art_search(search_index.at(field_name), key, key_len);
if(leaf != nullptr) {
- uint32_t doc_index = leaf->values->ids.indexOf(seq_id);
-
- if (doc_index == leaf->values->ids.getLength()) {
- // not found - happens when 2 tokens repeat in a field, e.g "is it or is is not?"
- continue;
- }
-
- uint32_t start_offset = leaf->values->offset_index.at(doc_index);
- uint32_t end_offset = (doc_index == leaf->values->ids.getLength() - 1) ?
- leaf->values->offsets.getLength() :
- leaf->values->offset_index.at(doc_index + 1);
-
- uint32_t doc_indices[1] = {doc_index};
- remove_and_shift_offset_index(leaf->values->offset_index, doc_indices, 1);
-
- leaf->values->offsets.remove_index(start_offset, end_offset);
- leaf->values->ids.remove_value(seq_id);
-
- /*len = leaf->values->offset_index.getLength();
- for(auto i=0; i<len; i++) {
- LOG(INFO) << "i: " << i << ", val: " << leaf->values->offset_index.at(i);
- }
- LOG(INFO) << "----";*/
-
- if (leaf->values->ids.getLength() == 0) {
- art_values *values = (art_values *) art_delete(search_index.at(field_name), key, key_len);
- delete values;
+ posting_t::erase(leaf->values, seq_id);
+ if (posting_t::num_ids(leaf->values) == 0) {
+ void* values = art_delete(search_index.at(field_name), key, key_len);
+ posting_t::destroy_list(values);
}
}
}
diff --git a/test/art_test.cpp b/test/art_test.cpp
index f81a9330..157b58dc 100644
--- a/test/art_test.cpp
+++ b/test/art_test.cpp
@@ -6,6 +6,7 @@
#include <gtest/gtest.h>
#include <art.h>
#include <chrono>
+#include <posting.h>
#define words_file_path std::string(std::string(ROOT_DIR)+"/build/test_resources/words.txt").c_str()
#define uuid_file_path std::string(std::string(ROOT_DIR)+"/build/test_resources/uuid.txt").c_str()
@@ -16,9 +17,7 @@ art_document get_document(uint32_t id) {
art_document document;
document.score = id;
document.id = id;
- document.offsets = new uint32_t[1]{0};
- document.offsets_len = 1;
-
+ document.offsets = {0};
return document;
}
@@ -48,11 +47,9 @@ TEST(ArtTest, test_art_insert) {
len = strlen(buf);
buf[len-1] = '\0';
art_document document = get_document(line);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)buf, len, &document, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)buf, len, &document));
ASSERT_TRUE(art_size(&t) == line);
line++;
-
- delete [] document.offsets;
}
res = art_tree_destroy(&t);
@@ -105,9 +102,9 @@ TEST(ArtTest, test_art_insert_verylong) {
art_document doc1 = get_document(1);
art_document doc2 = get_document(2);
- ASSERT_TRUE(NULL == art_insert(&t, key1, 299, &doc1, 1));
- ASSERT_TRUE(NULL == art_insert(&t, key2, 302, &doc2, 2));
- art_insert(&t, key2, 302, &doc2, 2);
+ ASSERT_TRUE(NULL == art_insert(&t, key1, 299, &doc1));
+ ASSERT_TRUE(NULL == art_insert(&t, key2, 302, &doc2));
+ art_insert(&t, key2, 302, &doc2);
EXPECT_EQ(art_size(&t), 2);
res = art_tree_destroy(&t);
@@ -128,7 +125,7 @@ TEST(ArtTest, test_art_insert_search) {
len = strlen(buf);
buf[len-1] = '\0';
art_document doc = get_document((uint32_t) line);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)buf, len, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)buf, len, &doc));
line++;
}
@@ -142,7 +139,7 @@ TEST(ArtTest, test_art_insert_search) {
buf[len-1] = '\0';
art_leaf* l = (art_leaf *) art_search(&t, (unsigned char*)buf, len);
- EXPECT_EQ(line, l->values->ids.at(0));
+ EXPECT_EQ(line, posting_t::first_id(l->values));
line++;
}
@@ -173,7 +170,7 @@ TEST(ArtTest, test_art_insert_delete) {
len = strlen(buf);
buf[len-1] = '\0';
art_document doc = get_document((uint32_t) line);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)buf, len, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)buf, len, &doc));
line++;
}
@@ -191,12 +188,12 @@ TEST(ArtTest, test_art_insert_delete) {
// Search first, ensure all entries still
// visible
art_leaf* l = (art_leaf *) art_search(&t, (unsigned char*)buf, len);
- EXPECT_EQ(line, l->values->ids.at(0));
+ EXPECT_EQ(line, posting_t::first_id(l->values));
// Delete, should get lineno back
- art_values* values = (art_values*)art_delete(&t, (unsigned char*)buf, len);
- EXPECT_EQ(line, values->ids.at(0));
- delete values;
+ void* values = (art_values*)art_delete(&t, (unsigned char*)buf, len);
+ EXPECT_EQ(line, posting_t::first_id(values));
+ posting_t::destroy_list(values);
// Check the size
ASSERT_TRUE(art_size(&t) == nlines - line);
@@ -214,7 +211,7 @@ TEST(ArtTest, test_art_insert_delete) {
int iter_cb(void *data, const unsigned char* key, uint32_t key_len, void *val) {
uint64_t *out = (uint64_t*)data;
- uintptr_t line = ((art_values*)val)->ids.at(0);
+ uintptr_t line = posting_t::first_id(val);
uint64_t mask = (line * (key[0] + key_len));
out[0]++;
out[1] ^= mask;
@@ -222,35 +219,35 @@ int iter_cb(void *data, const unsigned char* key, uint32_t key_len, void *val) {
}
TEST(ArtTest, test_art_insert_iter) {
- art_tree t;
- int res = art_tree_init(&t);
- ASSERT_TRUE(res == 0);
+ art_tree t;
+ int res = art_tree_init(&t);
+ ASSERT_TRUE(res == 0);
- int len;
- char buf[512];
- FILE *f = fopen(words_file_path, "r");
+ int len;
+ char buf[512];
+ FILE *f = fopen(words_file_path, "r");
- uint64_t xor_mask = 0;
- uintptr_t line = 1, nlines;
- while (fgets(buf, sizeof buf, f)) {
- len = strlen(buf);
- buf[len-1] = '\0';
- art_document doc = get_document((uint32_t) line);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)buf, len, &doc, 1));
+ uint64_t xor_mask = 0;
+ uintptr_t line = 1, nlines;
+ while (fgets(buf, sizeof buf, f)) {
+ len = strlen(buf);
+ buf[len-1] = '\0';
+ art_document doc = get_document((uint32_t) line);
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)buf, len, &doc));
- xor_mask ^= (line * (buf[0] + len));
- line++;
- }
- nlines = line - 1;
+ xor_mask ^= (line * (buf[0] + len));
+ line++;
+ }
+ nlines = line - 1;
- uint64_t out[] = {0, 0};
- ASSERT_TRUE(art_iter(&t, iter_cb, &out) == 0);
+ uint64_t out[] = {0, 0};
+ ASSERT_TRUE(art_iter(&t, iter_cb, &out) == 0);
- ASSERT_TRUE(out[0] == nlines);
- ASSERT_TRUE(out[1] == xor_mask);
+ ASSERT_TRUE(out[0] == nlines);
+ ASSERT_TRUE(out[1] == xor_mask);
- res = art_tree_destroy(&t);
- ASSERT_TRUE(res == 0);
+ res = art_tree_destroy(&t);
+ ASSERT_TRUE(res == 0);
}
@@ -275,27 +272,27 @@ TEST(ArtTest, test_art_iter_prefix) {
const char *s = "api.foo.bar";
art_document doc = get_document((uint32_t) 1);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc));
s = "api.foo.baz";
doc = get_document((uint32_t) 2);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc));
s = "api.foe.fum";
doc = get_document((uint32_t) 3);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc));
s = "abc.123.456";
doc = get_document((uint32_t) 4);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc));
s = "api.foo";
doc = get_document((uint32_t) 5);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc));
s = "api";
doc = get_document((uint32_t) 6);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc));
// Iterate over api
const char *expected[] = {"api", "api.foe.fum", "api.foo", "api.foo.bar", "api.foo.baz"};
@@ -352,27 +349,27 @@ TEST(ArtTest, test_art_long_prefix) {
s = "this:key:has:a:long:prefix:3";
id = 3;
art_document doc = get_document((uint32_t) id);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc));
s = "this:key:has:a:long:common:prefix:2";
id = 2;
doc = get_document((uint32_t) id);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc));
s = "this:key:has:a:long:common:prefix:1";
id = 1;
doc = get_document((uint32_t) id);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc));
// Search for the keys
s = "this:key:has:a:long:common:prefix:1";
- EXPECT_EQ(1, (((art_leaf *)art_search(&t, (unsigned char*)s, strlen(s)+1))->values->ids.at(0)));
+ EXPECT_EQ(1, posting_t::first_id(((art_leaf *)art_search(&t, (unsigned char*)s, strlen(s)+1))->values));
s = "this:key:has:a:long:common:prefix:2";
- EXPECT_EQ(2, (((art_leaf *)art_search(&t, (unsigned char*)s, strlen(s)+1))->values->ids.at(0)));
+ EXPECT_EQ(2, posting_t::first_id(((art_leaf *)art_search(&t, (unsigned char*)s, strlen(s)+1))->values));
s = "this:key:has:a:long:prefix:3";
- EXPECT_EQ(3, (((art_leaf *)art_search(&t, (unsigned char*)s, strlen(s)+1))->values->ids.at(0)));
+ EXPECT_EQ(3, posting_t::first_id(((art_leaf *)art_search(&t, (unsigned char*)s, strlen(s)+1))->values));
const char *expected[] = {
"this:key:has:a:long:common:prefix:1",
@@ -401,7 +398,7 @@ TEST(ArtTest, test_art_insert_search_uuid) {
len = strlen(buf);
buf[len-1] = '\0';
art_document doc = get_document((uint32_t) line);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)buf, len, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)buf, len, &doc));
line++;
}
@@ -414,7 +411,7 @@ TEST(ArtTest, test_art_insert_search_uuid) {
len = strlen(buf);
buf[len-1] = '\0';
- uintptr_t id = ((art_leaf*)art_search(&t, (unsigned char*)buf, len))->values->ids.at(0);
+ uintptr_t id = posting_t::first_id(((art_leaf*)art_search(&t, (unsigned char*)buf, len))->values);
ASSERT_TRUE(line == id);
line++;
}
@@ -438,15 +435,15 @@ TEST(ArtTest, test_art_max_prefix_len_scan_prefix) {
const char* key1 = "foobarbaz1-test1-foo";
art_document doc = get_document((uint32_t) 1);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key1, strlen(key1)+1, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key1, strlen(key1)+1, &doc));
const char *key2 = "foobarbaz1-test1-bar";
doc = get_document((uint32_t) 2);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key2, strlen(key2)+1, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key2, strlen(key2)+1, &doc));
const char *key3 = "foobarbaz1-test2-foo";
doc = get_document((uint32_t) 3);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key3, strlen(key3)+1, &doc, 2));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key3, strlen(key3)+1, &doc));
ASSERT_TRUE(art_size(&t) == 3);
@@ -469,15 +466,15 @@ TEST(ArtTest, test_art_prefix_iter_out_of_bounds) {
const char* key1 = "foobarbaz1-long-test1-foo";
art_document doc = get_document((uint32_t) 1);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key1, strlen(key1)+1, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key1, strlen(key1)+1, &doc));
const char *key2 = "foobarbaz1-long-test1-bar";
doc = get_document((uint32_t) 2);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key2, strlen(key2)+1, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key2, strlen(key2)+1, &doc));
const char *key3 = "foobarbaz1-long-test2-foo";
doc = get_document((uint32_t) 3);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key3, strlen(key3)+1, &doc, 2));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key3, strlen(key3)+1, &doc));
ASSERT_TRUE(art_size(&t) == 3);
@@ -500,15 +497,15 @@ TEST(ArtTest, test_art_search_out_of_bounds) {
const char* key1 = "foobarbaz1-long-test1-foo";
art_document doc = get_document((uint32_t) 1);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key1, strlen(key1)+1, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key1, strlen(key1)+1, &doc));
const char *key2 = "foobarbaz1-long-test1-bar";
doc = get_document((uint32_t) 2);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key2, strlen(key2)+1, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key2, strlen(key2)+1, &doc));
const char *key3 = "foobarbaz1-long-test2-foo";
doc = get_document((uint32_t) 3);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key3, strlen(key3)+1, &doc, 2));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key3, strlen(key3)+1, &doc));
ASSERT_TRUE(art_size(&t) == 3);
@@ -529,15 +526,15 @@ TEST(ArtTest, test_art_delete_out_of_bounds) {
const char* key1 = "foobarbaz1-long-test1-foo";
art_document doc = get_document((uint32_t) 1);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key1, strlen(key1)+1, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key1, strlen(key1)+1, &doc));
const char *key2 = "foobarbaz1-long-test1-bar";
doc = get_document((uint32_t) 2);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key2, strlen(key2)+1, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key2, strlen(key2)+1, &doc));
const char *key3 = "foobarbaz1-long-test2-foo";
doc = get_document((uint32_t) 3);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key3, strlen(key3)+1, &doc, 2));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key3, strlen(key3)+1, &doc));
ASSERT_TRUE(art_size(&t) == 3);
@@ -557,20 +554,24 @@ TEST(ArtTest, test_art_insert_multiple_ids_for_same_token) {
const char* key1 = "implement";
art_document doc = get_document((uint32_t) 1);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key1, strlen(key1)+1, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key1, strlen(key1)+1, &doc));
doc = get_document((uint32_t) 2);
- art_values* value = (art_values*) art_insert(&t, (unsigned char*)key1, strlen(key1) + 1, &doc, 2);
+ void* value = art_insert(&t, (unsigned char*)key1, strlen(key1) + 1, &doc);
ASSERT_TRUE(value != NULL);
+ ASSERT_EQ(posting_t::num_ids(value), 2);
+ ASSERT_EQ(posting_t::first_id(value), 1);
+ ASSERT_TRUE(posting_t::contains(value, 2));
+
doc = get_document((uint32_t) 3);
- ASSERT_TRUE(value == (art_values*) art_insert(&t, (unsigned char*)key1, strlen(key1)+1, &doc, 3));
+ void* reinsert_value = art_insert(&t, (unsigned char*) key1, strlen(key1) + 1, &doc);
ASSERT_TRUE(art_size(&t) == 1);
- ASSERT_EQ(value->ids.getLength(), 3);
- ASSERT_EQ(value->ids.at(0), 1);
- ASSERT_EQ(value->ids.at(1), 2);
- ASSERT_EQ(value->ids.at(2), 3);
+ ASSERT_EQ(posting_t::num_ids(reinsert_value), 3);
+ ASSERT_EQ(posting_t::first_id(reinsert_value), 1);
+ ASSERT_TRUE(posting_t::contains(reinsert_value, 2));
+ ASSERT_TRUE(posting_t::contains(reinsert_value, 3));
res = art_tree_destroy(&t);
ASSERT_TRUE(res == 0);
@@ -583,10 +584,10 @@ TEST(ArtTest, test_art_fuzzy_search_single_leaf) {
const char* implement_key = "implement";
art_document doc = get_document((uint32_t) 1);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)implement_key, strlen(implement_key)+1, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)implement_key, strlen(implement_key)+1, &doc));
art_leaf* l = (art_leaf *) art_search(&t, (const unsigned char *)implement_key, strlen(implement_key)+1);
- EXPECT_EQ(1, l->values->ids.at(0));
+ EXPECT_EQ(1, posting_t::first_id(l->values));
std::vector<art_leaf*> leaves;
art_fuzzy_search(&t, (const unsigned char *) implement_key, strlen(implement_key) + 1, 0, 0, 10, FREQUENCY, false, nullptr, 0, leaves);
@@ -618,10 +619,10 @@ TEST(ArtTest, test_art_fuzzy_search_single_leaf_prefix) {
const char* key = "application";
art_document doc = get_document((uint32_t) 1);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key, strlen(key)+1, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key, strlen(key)+1, &doc));
art_leaf* l = (art_leaf *) art_search(&t, (const unsigned char *)key, strlen(key)+1);
- EXPECT_EQ(1, l->values->ids.at(0));
+ EXPECT_EQ(1, posting_t::first_id(l->values));
std::vector<art_leaf*> leaves;
art_fuzzy_search(&t, (const unsigned char *) "aplication", strlen(key)-1, 0, 1, 10, FREQUENCY, true, nullptr, 0, leaves);
@@ -649,7 +650,7 @@ TEST(ArtTest, test_art_fuzzy_search) {
len = strlen(buf);
buf[len-1] = '\0';
art_document doc = get_document((uint32_t) line);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)buf, len, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)buf, len, &doc));
line++;
}
@@ -757,12 +758,12 @@ TEST(ArtTest, test_art_fuzzy_search_unicode_chars) {
for(const char* key: keys) {
art_document doc = get_document((uint32_t) 1);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key, strlen(key)+1, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key, strlen(key)+1, &doc));
}
for(const char* key: keys) {
art_leaf* l = (art_leaf *) art_search(&t, (const unsigned char *)key, strlen(key)+1);
- EXPECT_EQ(1, l->values->ids.at(0));
+ EXPECT_EQ(1, posting_t::first_id(l->values));
std::vector<art_leaf*> leaves;
art_fuzzy_search(&t, (unsigned char *)key, strlen(key), 0, 0, 10, FREQUENCY, true, nullptr, 0, leaves);
@@ -789,7 +790,7 @@ TEST(ArtTest, test_art_search_sku_like_tokens) {
len = strlen(buf);
buf[len - 1] = '\0';
art_document doc = get_document((uint32_t) line);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) buf, len, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) buf, len, &doc));
keys.push_back(std::string(buf, len-1));
line++;
}
@@ -798,7 +799,7 @@ TEST(ArtTest, test_art_search_sku_like_tokens) {
// exact search
art_leaf* l = (art_leaf *) art_search(&t, (const unsigned char *)key1, strlen(key1)+1);
- EXPECT_EQ(1, l->values->ids.getLength());
+ EXPECT_EQ(1, posting_t::num_ids(l->values));
// exact search all tokens via fuzzy API
@@ -838,7 +839,7 @@ TEST(ArtTest, test_art_search_ill_like_tokens) {
len = strlen(buf);
buf[len - 1] = '\0';
art_document doc = get_document((uint32_t) line);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) buf, len, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) buf, len, &doc));
keys.push_back(std::string(buf, len-1));
line++;
}
@@ -860,7 +861,7 @@ TEST(ArtTest, test_art_search_ill_like_tokens) {
for (const auto &key : keys) {
art_leaf* l = (art_leaf *) art_search(&t, (const unsigned char *)key.c_str(), key.size()+1);
ASSERT_FALSE(l == nullptr);
- EXPECT_EQ(1, l->values->ids.getLength());
+ EXPECT_EQ(1, posting_t::num_ids(l->values));
std::vector<art_leaf *> leaves;
art_fuzzy_search(&t, (const unsigned char*)key.c_str(), key.size(), 0, 0, 10,
@@ -895,18 +896,18 @@ TEST(ArtTest, test_art_search_ill_like_tokens2) {
keys = {"input", "illustrations", "illustration"};
art_document doc = get_document((uint32_t) 1);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) keys[0].c_str(), keys[0].size()+1, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) keys[0].c_str(), keys[0].size()+1, &doc));
doc = get_document((uint32_t) 2);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) keys[1].c_str(), keys[1].size()+1, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) keys[1].c_str(), keys[1].size()+1, &doc));
doc = get_document((uint32_t) 3);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) keys[2].c_str(), keys[2].size()+1, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) keys[2].c_str(), keys[2].size()+1, &doc));
for (const auto &key : keys) {
art_leaf* l = (art_leaf *) art_search(&t, (const unsigned char *)key.c_str(), key.size()+1);
ASSERT_FALSE(l == nullptr);
- EXPECT_EQ(1, l->values->ids.getLength());
+ EXPECT_EQ(1, posting_t::num_ids(l->values));
std::vector<art_leaf *> leaves;
art_fuzzy_search(&t, (const unsigned char*)key.c_str(), key.size(), 0, 0, 10,
@@ -941,7 +942,7 @@ TEST(ArtTest, test_art_search_roche_chews) {
keys = {"roche"};
art_document doc = get_document((uint32_t) 1);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) keys[0].c_str(), keys[0].size()+1, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) keys[0].c_str(), keys[0].size()+1, &doc));
std::string term = "chews";
std::vector<art_leaf *> leaves;
@@ -969,7 +970,7 @@ TEST(ArtTest, test_art_search_raspberry) {
for (const auto &key : keys) {
art_document doc = get_document((uint32_t) 1);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) key.c_str(), key.size()+1, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) key.c_str(), key.size()+1, &doc));
}
// prefix search
@@ -1002,7 +1003,7 @@ TEST(ArtTest, test_art_search_highliving) {
for (const auto &key : keys) {
art_document doc = get_document((uint32_t) 1);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) key.c_str(), key.size()+1, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) key.c_str(), key.size()+1, &doc));
}
// prefix search
@@ -1086,7 +1087,7 @@ TEST(ArtTest, test_int32_overlap) {
for(size_t j = 0; j < values[i].size(); j++) {
encode_int32(values[i][j], chars);
doc.id = i;
- art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc, 1);
+ art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc);
}
}
@@ -1110,7 +1111,7 @@ TEST(ArtTest, test_int32_range_hundreds) {
for(uint32_t i = 100; i < 110; i++) {
encode_int32(i, chars);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc));
}
encode_int32(106, chars);
@@ -1156,7 +1157,7 @@ TEST(ArtTest, test_int32_duplicates) {
doc.id = i;
int value = 1900 + (rand() % static_cast<int>(2018 - 1900 + 1));
encode_int32(value, chars);
- art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc, 1);
+ art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc);
}
std::vector<const art_leaf*> results;
@@ -1166,7 +1167,7 @@ TEST(ArtTest, test_int32_duplicates) {
size_t counter = 0;
for(auto res: results) {
- counter += res->values->ids.getLength();
+ counter += posting_t::num_ids(res->values);
}
ASSERT_EQ(10000, counter);
@@ -1183,7 +1184,7 @@ TEST(ArtTest, test_int32_negative) {
for(int32_t i = -100; i < 0; i++) {
encode_int32(i, chars);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc));
}
encode_int32(-99, chars);
@@ -1235,7 +1236,7 @@ TEST(ArtTest, test_int32_million) {
for(uint32_t i = 0; i < 1000000; i++) {
encode_int32(i, chars);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc));
}
encode_int32(5, chars);
@@ -1314,7 +1315,7 @@ TEST(ArtTest, test_int_range_byte_boundary) {
for(uint32_t i = 200; i < 300; i++) {
encode_int32(i, chars);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc));
}
encode_int32(255, chars);
@@ -1376,7 +1377,7 @@ TEST(ArtTest, test_search_int64) {
for(uint64_t i = lmax; i < lmax+100; i++) {
encode_int64(i, chars);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc));
}
std::vector<const art_leaf*> results;
@@ -1419,7 +1420,7 @@ TEST(ArtTest, test_search_negative_int64) {
for(int64_t i = lmax-100; i < lmax; i++) {
encode_int64(i, chars);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc));
}
std::vector<const art_leaf*> results;
@@ -1458,10 +1459,8 @@ TEST(ArtTest, test_search_negative_int64_large) {
const int CHAR_LEN = 8;
unsigned char chars[CHAR_LEN];
- const int64_t lmax = -1 * std::numeric_limits<std::int32_t>::max();
-
encode_int64(-2, chars);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) chars, CHAR_LEN, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) chars, CHAR_LEN, &doc));
std::vector<const art_leaf *> results;
@@ -1491,7 +1490,7 @@ TEST(ArtTest, test_int32_array) {
for (size_t j = 0; j < values[i].size(); j++) {
encode_int32(values[i][j], chars);
doc.id = i;
- art_insert(&t, (unsigned char *) chars, CHAR_LEN, &doc, 1);
+ art_insert(&t, (unsigned char *) chars, CHAR_LEN, &doc);
}
}
@@ -1511,7 +1510,7 @@ TEST(ArtTest, test_encode_float_positive) {
unsigned char chars0[CHAR_LEN];
encode_float(floats[i], chars0);
art_document doc = get_document(i);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars0, CHAR_LEN, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars0, CHAR_LEN, &doc));
}
std::vector<const art_leaf*> results;
@@ -1568,7 +1567,7 @@ TEST(ArtTest, test_encode_float_positive_negative) {
unsigned char chars0[CHAR_LEN];
encode_float(floats[i], chars0);
art_document doc = get_document(i);
- ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars0, CHAR_LEN, &doc, 1));
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars0, CHAR_LEN, &doc));
}
std::vector<const art_leaf*> results;
diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cpp
index 9f77ac27..e6d7f17c 100644
--- a/test/collection_all_fields_test.cpp
+++ b/test/collection_all_fields_test.cpp
@@ -505,6 +505,8 @@ TEST_F(CollectionAllFieldsTest, StringSingularAllValues) {
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("FIRST", results["hits"][0]["document"]["title"].get<std::string>());
ASSERT_EQ("123", results["hits"][0]["document"]["int_values"].get<std::string>());
+
+ collectionManager.drop_collection("coll1");
}
TEST_F(CollectionAllFieldsTest, UpdateOfDocumentsInAutoMode) {
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index fb87efab..dab4a2f9 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -75,7 +75,7 @@ protected:
std::vector<std::string> strs;
for(size_t i = 0 ; i < num_words ; i++ ) {
- int word_index = rand() % 100;
+ int word_index = rand() % words.size();
strs.push_back(words[word_index]);
}
return StringUtils::join(strs, " ");
@@ -1182,6 +1182,50 @@ TEST_F(CollectionTest, ImportDocumentsUpsert) {
ASSERT_EQ(409, import_results[1]["code"].get<size_t>());
}
+TEST_F(CollectionTest, DISABLED_CrashTroubleshooting) {
+ Collection *coll1;
+ std::vector<field> fields = {
+ field("title", field_types::STRING_ARRAY, false, true),
+ field("points", field_types::INT32, false)
+ };
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
+ }
+
+ std::ifstream create_file("/tmp/create.jsonl");
+ std::string json_line;
+ std::vector<std::string> create_records;
+
+ while (std::getline(create_file, json_line)) {
+ create_records.push_back(json_line);
+ }
+
+ create_file.close();
+
+ nlohmann::json document;
+ auto import_response = coll1->add_many(create_records, document, CREATE);
+
+ ASSERT_TRUE(import_response["success"].get<bool>());
+ ASSERT_EQ(1000, import_response["num_imported"].get<int>());
+
+ // now try to upsert
+
+ std::ifstream upsert_file("/tmp/upsert.jsonl");
+ std::vector<std::string> upsert_records;
+
+ while (std::getline(upsert_file, json_line)) {
+ upsert_records.push_back(json_line);
+ }
+
+ upsert_file.close();
+
+ import_response = coll1->add_many(upsert_records, document, UPSERT);
+
+ ASSERT_TRUE(import_response["success"].get<bool>());
+ ASSERT_EQ(1000, import_response["num_imported"].get<int>());
+}
TEST_F(CollectionTest, ImportDocumentsUpsertOptional) {
Collection *coll1;