Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Compiles, but tests are failing.

Kishore Nallanf347375adb5a
diff --git a/include/collection.h b/include/collection.h
index 21954c0b..b2e15a8a 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -3,6 +3,7 @@
#include <string>
#include <vector>
#include <art.h>
+#include <index.h>
#include <number.h>
#include <sparsepp.h>
#include <store.h>
@@ -11,33 +12,6 @@
#include <field.h>
#include <option.h>
-struct facet_value {
- // use string to int mapping for saving memory
- spp::sparse_hash_map<std::string, uint32_t> value_index;
- spp::sparse_hash_map<uint32_t, std::string> index_value;
-
- spp::sparse_hash_map<uint32_t, std::vector<uint32_t>> doc_values;
-
- uint32_t get_value_index(const std::string & value) {
- if(value_index.count(value) != 0) {
- return value_index[value];
- }
-
- uint32_t new_index = value_index.size();
- value_index.emplace(value, new_index);
- index_value.emplace(new_index, value);
- return new_index;
- }
-
- void index_values(uint32_t doc_seq_id, const std::vector<std::string> & values) {
- std::vector<uint32_t> value_vec(values.size());
- for(auto i = 0; i < values.size(); i++) {
- value_vec[i] = get_value_index(values[i]);
- }
- doc_values.emplace(doc_seq_id, value_vec);
- }
-};
-
class Collection {
private:
std::string name;
@@ -46,6 +20,10 @@ private:
size_t num_documents;
+ size_t num_indices;
+
+ std::vector<Index*> indices;
+
// Auto incrementing record ID used internally for indexing - not exposed to the client
uint32_t next_seq_id;
@@ -57,67 +35,12 @@ private:
Store* store;
- spp::sparse_hash_map<std::string, art_tree*> search_index;
-
- spp::sparse_hash_map<std::string, facet_value> facet_index;
-
- spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, number_t>*> sort_index;
-
std::string token_ranking_field;
std::string get_doc_id_key(const std::string & doc_id);
std::string get_seq_id_key(uint32_t seq_id);
- static inline std::vector<art_leaf *> next_suggestion(const std::vector<std::vector<art_leaf *>> &token_leaves,
- long long int n);
-
- void log_leaves(const int cost, const std::string &token, const std::vector<art_leaf *> &leaves) const;
-
- size_t union_of_ids(std::vector<std::pair<uint32_t*, size_t>> & result_array_pairs, uint32_t **results_out);
-
- Option<uint32_t> do_filtering(uint32_t** filter_ids_out, const std::string & simple_filter_str);
-
- void do_facets(std::vector<facet> & facets, uint32_t* result_ids, size_t results_size);
-
- void populate_token_positions(const std::vector<art_leaf *> &query_suggestion,
- spp::sparse_hash_map<const art_leaf *, uint32_t *> &leaf_to_indices,
- size_t result_index, std::vector<std::vector<uint16_t>> &token_positions) const;
-
- void search_field(std::string & query, const std::string & field, uint32_t *filter_ids, size_t filter_ids_length,
- std::vector<facet> & facets, const std::vector<sort_by> & sort_fields,
- const int num_typos, const size_t num_results,
- std::vector<std::vector<art_leaf*>> & searched_queries, int & searched_queries_index,
- Topster<100> & topster, uint32_t** all_result_ids,
- size_t & all_result_ids_len, const token_ordering token_order = FREQUENCY, const bool prefix = false);
-
- void search_candidates(uint32_t* filter_ids, size_t filter_ids_length, std::vector<facet> & facets,
- const std::vector<sort_by> & sort_fields, int & candidate_rank,
- std::vector<std::vector<art_leaf*>> & token_to_candidates,
- std::vector<std::vector<art_leaf*>> & searched_queries, Topster<100> & topster,
- size_t & total_results, uint32_t** all_result_ids, size_t & all_result_ids_len,
- const size_t & max_results, const bool prefix);
-
- void index_string_field(const std::string & text, const uint32_t score, art_tree *t, uint32_t seq_id,
- const bool verbatim) const;
-
- void index_string_array_field(const std::vector<std::string> & strings, const uint32_t score, art_tree *t,
- uint32_t seq_id, const bool verbatim) const;
-
- void index_int32_field(const int32_t value, const uint32_t score, art_tree *t, uint32_t seq_id) const;
-
- void index_int64_field(const int64_t value, const uint32_t score, art_tree *t, uint32_t seq_id) const;
-
- void index_float_field(const float value, const uint32_t score, art_tree *t, uint32_t seq_id) const;
-
- void index_int32_array_field(const std::vector<int32_t> & values, const uint32_t score, art_tree *t, uint32_t seq_id) const;
-
- void index_int64_array_field(const std::vector<int64_t> & values, const uint32_t score, art_tree *t, uint32_t seq_id) const;
-
- void index_float_array_field(const std::vector<float> & values, const uint32_t score, art_tree *t, uint32_t seq_id) const;
-
- void remove_and_shift_offset_index(sorted_array &offset_index, const uint32_t *indices_sorted,
- const uint32_t indices_length);
Option<uint32_t> validate_index_in_memory(const nlohmann::json &document, uint32_t seq_id);
public:
@@ -170,10 +93,6 @@ public:
Option<std::string> remove(const std::string & id, const bool remove_from_store = true);
- void score_results(const std::vector<sort_by> & sort_fields, const int & query_index, const int & candidate_rank,
- Topster<100> &topster, const std::vector<art_leaf *> & query_suggestion, const uint32_t *result_ids,
- const size_t result_size) const;
-
Option<uint32_t> index_in_memory(const nlohmann::json & document, uint32_t seq_id);
static const int MAX_SEARCH_TOKENS = 10;
diff --git a/include/field.h b/include/field.h
index 583d278d..da0745ce 100644
--- a/include/field.h
+++ b/include/field.h
@@ -117,4 +117,31 @@ struct facet {
facet(const std::string field_name): field_name(field_name) {
}
+};
+
+struct facet_value {
+ // use string to int mapping for saving memory
+ spp::sparse_hash_map<std::string, uint32_t> value_index;
+ spp::sparse_hash_map<uint32_t, std::string> index_value;
+
+ spp::sparse_hash_map<uint32_t, std::vector<uint32_t>> doc_values;
+
+ uint32_t get_value_index(const std::string & value) {
+ if(value_index.count(value) != 0) {
+ return value_index[value];
+ }
+
+ uint32_t new_index = value_index.size();
+ value_index.emplace(value, new_index);
+ index_value.emplace(new_index, value);
+ return new_index;
+ }
+
+ void index_values(uint32_t doc_seq_id, const std::vector<std::string> & values) {
+ std::vector<uint32_t> value_vec(values.size());
+ for(auto i = 0; i < values.size(); i++) {
+ value_vec[i] = get_value_index(values[i]);
+ }
+ doc_values.emplace(doc_seq_id, value_vec);
+ }
};
\ No newline at end of file
diff --git a/include/index.h b/include/index.h
index 21954c0b..ff88b6a0 100644
--- a/include/index.h
+++ b/include/index.h
@@ -11,34 +11,7 @@
#include <field.h>
#include <option.h>
-struct facet_value {
- // use string to int mapping for saving memory
- spp::sparse_hash_map<std::string, uint32_t> value_index;
- spp::sparse_hash_map<uint32_t, std::string> index_value;
-
- spp::sparse_hash_map<uint32_t, std::vector<uint32_t>> doc_values;
-
- uint32_t get_value_index(const std::string & value) {
- if(value_index.count(value) != 0) {
- return value_index[value];
- }
-
- uint32_t new_index = value_index.size();
- value_index.emplace(value, new_index);
- index_value.emplace(new_index, value);
- return new_index;
- }
-
- void index_values(uint32_t doc_seq_id, const std::vector<std::string> & values) {
- std::vector<uint32_t> value_vec(values.size());
- for(auto i = 0; i < values.size(); i++) {
- value_vec[i] = get_value_index(values[i]);
- }
- doc_values.emplace(doc_seq_id, value_vec);
- }
-};
-
-class Collection {
+class Index {
private:
std::string name;
@@ -46,29 +19,18 @@ private:
size_t num_documents;
- // Auto incrementing record ID used internally for indexing - not exposed to the client
- uint32_t next_seq_id;
-
spp::sparse_hash_map<std::string, field> search_schema;
spp::sparse_hash_map<std::string, field> facet_schema;
spp::sparse_hash_map<std::string, field> sort_schema;
- Store* store;
-
spp::sparse_hash_map<std::string, art_tree*> search_index;
spp::sparse_hash_map<std::string, facet_value> facet_index;
spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, number_t>*> sort_index;
- std::string token_ranking_field;
-
- std::string get_doc_id_key(const std::string & doc_id);
-
- std::string get_seq_id_key(uint32_t seq_id);
-
static inline std::vector<art_leaf *> next_suggestion(const std::vector<std::vector<art_leaf *>> &token_leaves,
long long int n);
@@ -87,7 +49,7 @@ private:
void search_field(std::string & query, const std::string & field, uint32_t *filter_ids, size_t filter_ids_length,
std::vector<facet> & facets, const std::vector<sort_by> & sort_fields,
const int num_typos, const size_t num_results,
- std::vector<std::vector<art_leaf*>> & searched_queries, int & searched_queries_index,
+ std::vector<std::vector<art_leaf*>> & searched_queries,
Topster<100> & topster, uint32_t** all_result_ids,
size_t & all_result_ids_len, const token_ordering token_order = FREQUENCY, const bool prefix = false);
@@ -118,63 +80,30 @@ private:
void remove_and_shift_offset_index(sorted_array &offset_index, const uint32_t *indices_sorted,
const uint32_t indices_length);
- Option<uint32_t> validate_index_in_memory(const nlohmann::json &document, uint32_t seq_id);
public:
- Collection() = delete;
-
- Collection(const std::string name, const uint32_t collection_id, const uint32_t next_seq_id, Store *store,
- const std::vector<field> & fields, const std::string & token_ranking_field);
-
- ~Collection();
-
- static std::string get_next_seq_id_key(const std::string & collection_name);
-
- static std::string get_meta_key(const std::string & collection_name);
-
- std::string get_seq_id_collection_prefix();
-
- std::string get_name();
-
- size_t get_num_documents();
-
- uint32_t get_collection_id();
-
- uint32_t get_next_seq_id();
-
- void set_next_seq_id(uint32_t seq_id);
-
- void increment_next_seq_id_field();
-
- static uint32_t deserialize_seq_id_key(std::string serialized_seq_id);
-
- uint32_t doc_id_to_seq_id(std::string doc_id);
-
- std::vector<std::string> get_facet_fields();
-
- std::vector<field> get_sort_fields();
-
- spp::sparse_hash_map<std::string, field> get_schema();
-
- std::string get_token_ranking_field();
+ Index() = delete;
- Option<std::string> add(const std::string & json_str);
+ Index(const std::string name, spp::sparse_hash_map<std::string, field> search_schema,
+ spp::sparse_hash_map<std::string, field> facet_schema, spp::sparse_hash_map<std::string, field> sort_schema);
- Option<nlohmann::json> search(std::string query, const std::vector<std::string> search_fields,
- const std::string & simple_filter_query, const std::vector<std::string> & facet_fields,
- const std::vector<sort_by> & sort_fields, const int num_typos,
- const size_t per_page = 10, const size_t page = 1,
- const token_ordering token_order = FREQUENCY, const bool prefix = false);
+ ~Index();
- Option<nlohmann::json> get(const std::string & id);
+ Option<size_t> search(std::string query, const std::vector<std::string> search_fields,
+ const std::string & simple_filter_query, std::vector<facet> facets,
+ std::vector<sort_by> sort_fields_std, const int num_typos,
+ const size_t per_page, const size_t page,
+ const token_ordering token_order, const bool prefix,
+ std::vector<std::pair<int, Topster<100>::KV>> & field_order_kv,
+ size_t & all_result_ids_len, std::vector<std::vector<art_leaf*>> & searched_queries);
- Option<std::string> remove(const std::string & id, const bool remove_from_store = true);
+ Option<uint32_t> remove(const uint32_t seq_id, nlohmann::json & document);
void score_results(const std::vector<sort_by> & sort_fields, const int & query_index, const int & candidate_rank,
Topster<100> &topster, const std::vector<art_leaf *> & query_suggestion, const uint32_t *result_ids,
const size_t result_size) const;
- Option<uint32_t> index_in_memory(const nlohmann::json & document, uint32_t seq_id);
+ Option<uint32_t> index_in_memory(const nlohmann::json & document, uint32_t seq_id, int32_t points);
static const int MAX_SEARCH_TOKENS = 10;
static const int MAX_RESULTS = 100;
diff --git a/src/collection.cpp b/src/collection.cpp
index 2c548e5b..1178c062 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1,4 +1,4 @@
-#include "index.h"
+#include "collection.h"
#include <numeric>
#include <chrono>
@@ -13,42 +13,30 @@ Collection::Collection(const std::string name, const uint32_t collection_id, con
token_ranking_field(token_ranking_field) {
for(const field& field: fields) {
- art_tree *t = new art_tree;
- art_tree_init(t);
- search_index.emplace(field.name, t);
search_schema.emplace(field.name, field);
if(field.is_facet()) {
facet_value fvalue;
- facet_index.emplace(field.name, fvalue);
facet_schema.emplace(field.name, field);
}
if(field.is_single_integer() || field.is_single_float()) {
- spp::sparse_hash_map<uint32_t, number_t> * doc_to_score = new spp::sparse_hash_map<uint32_t, number_t>();
- sort_index.emplace(field.name, doc_to_score);
sort_schema.emplace(field.name, field);
}
}
+ num_indices = 4;
+ for(auto i = 0; i < num_indices; i++) {
+ indices.push_back(new Index(name, search_schema, facet_schema, sort_schema));
+ }
+
num_documents = 0;
}
Collection::~Collection() {
- for(auto & name_tree: search_index) {
- art_tree_destroy(name_tree.second);
- delete name_tree.second;
- name_tree.second = nullptr;
- }
-
- search_index.clear();
-
- for(auto & name_map: sort_index) {
- delete name_map.second;
- name_map.second = nullptr;
+ for(auto index: indices) {
+ delete index;
}
-
- sort_index.clear();
}
uint32_t Collection::get_next_seq_id() {
@@ -124,8 +112,6 @@ Option<uint32_t> Collection::validate_index_in_memory(const nlohmann::json &docu
"but is not found in the document.");
}
- art_tree *t = search_index.at(field_name);
-
if(field_pair.second.type == field_types::STRING) {
if(!document[field_name].is_string()) {
return Option<>(400, "Field `" + field_name + "` must be a string.");
@@ -188,7 +174,6 @@ Option<uint32_t> Collection::validate_index_in_memory(const nlohmann::json &docu
"but is not found in the document.");
}
- facet_value & fvalue = facet_index.at(field_name);
if(field_pair.second.type == field_types::STRING) {
if(!document[field_name].is_string()) {
return Option<>(400, "Facet field `" + field_name + "` must be a string.");
@@ -230,526 +215,18 @@ Option<uint32_t> Collection::index_in_memory(const nlohmann::json &document, uin
}
}
- for(const std::pair<std::string, field> & field_pair: search_schema) {
- const std::string & field_name = field_pair.first;
- art_tree *t = search_index.at(field_name);
-
- if(field_pair.second.type == field_types::STRING) {
- const std::string & text = document[field_name];
- index_string_field(text, points, t, seq_id, field_pair.second.is_facet());
- } else if(field_pair.second.type == field_types::INT32) {
- uint32_t value = document[field_name];
- index_int32_field(value, points, t, seq_id);
- } else if(field_pair.second.type == field_types::INT64) {
- uint64_t value = document[field_name];
- index_int64_field(value, points, t, seq_id);
- } else if(field_pair.second.type == field_types::FLOAT) {
- float value = document[field_name];
- index_float_field(value, points, t, seq_id);
- } else if(field_pair.second.type == field_types::STRING_ARRAY) {
- std::vector<std::string> strings = document[field_name];
- index_string_array_field(strings, points, t, seq_id, field_pair.second.is_facet());
- } else if(field_pair.second.type == field_types::INT32_ARRAY) {
- std::vector<int32_t> values = document[field_name];
- index_int32_array_field(values, points, t, seq_id);
- } else if(field_pair.second.type == field_types::INT64_ARRAY) {
- std::vector<int64_t> values = document[field_name];
- index_int64_array_field(values, points, t, seq_id);
- } else if(field_pair.second.type == field_types::FLOAT_ARRAY) {
- std::vector<float> values = document[field_name];
- index_float_array_field(values, points, t, seq_id);
- }
-
- // add numerical values automatically into sort index
- if(field_pair.second.type == field_types::INT32 || field_pair.second.type == field_types::INT64 ||
- field_pair.second.type == field_types::FLOAT) {
- spp::sparse_hash_map<uint32_t, number_t> *doc_to_score = sort_index.at(field_pair.first);
-
- if(document[field_pair.first].is_number_integer()) {
- doc_to_score->emplace(seq_id, document[field_pair.first].get<int64_t>());
- } else if(document[field_pair.first].is_number_float()) {
- doc_to_score->emplace(seq_id, document[field_pair.first].get<float>());
- }
- }
- }
-
- for(const std::pair<std::string, field> & field_pair: facet_schema) {
- const std::string & field_name = field_pair.first;
- facet_value & fvalue = facet_index.at(field_name);
- if(field_pair.second.type == field_types::STRING) {
- const std::string & value = document[field_name];
- fvalue.index_values(seq_id, { value });
- } else if(field_pair.second.type == field_types::STRING_ARRAY) {
- const std::vector<std::string> & values = document[field_name];
- fvalue.index_values(seq_id, values);
- }
- }
+ Index* index = indices[seq_id % num_indices];
+ index->index_in_memory(document, seq_id, points);
num_documents += 1;
return Option<>(200);
}
-void Collection::index_int32_field(const int32_t value, uint32_t score, art_tree *t, uint32_t seq_id) const {
- const int KEY_LEN = 8;
- unsigned char key[KEY_LEN];
-
- encode_int32(value, key);
-
- uint32_t num_hits = 0;
- art_leaf* leaf = (art_leaf *) art_search(t, key, KEY_LEN);
- if(leaf != NULL) {
- num_hits = leaf->values->ids.getLength();
- }
-
- num_hits += 1;
-
- art_document art_doc;
- art_doc.id = seq_id;
- art_doc.score = score;
- art_doc.offsets_len = 0;
- art_doc.offsets = nullptr;
-
- art_insert(t, key, KEY_LEN, &art_doc, num_hits);
-}
-
-void Collection::index_int64_field(const int64_t value, uint32_t score, art_tree *t, uint32_t seq_id) const {
- const int KEY_LEN = 8;
- unsigned char key[KEY_LEN];
-
- encode_int64(value, key);
-
- uint32_t num_hits = 0;
- art_leaf* leaf = (art_leaf *) art_search(t, key, KEY_LEN);
- if(leaf != NULL) {
- num_hits = leaf->values->ids.getLength();
- }
-
- num_hits += 1;
-
- art_document art_doc;
- art_doc.id = seq_id;
- art_doc.score = score;
- art_doc.offsets_len = 0;
- art_doc.offsets = nullptr;
-
- art_insert(t, key, KEY_LEN, &art_doc, num_hits);
-}
-
-void Collection::index_float_field(const float value, uint32_t score, art_tree *t, uint32_t seq_id) const {
- const int KEY_LEN = 8;
- unsigned char key[KEY_LEN];
-
- encode_float(value, key);
-
- uint32_t num_hits = 0;
- art_leaf* leaf = (art_leaf *) art_search(t, key, KEY_LEN);
- if(leaf != NULL) {
- num_hits = leaf->values->ids.getLength();
- }
-
- num_hits += 1;
-
- art_document art_doc;
- art_doc.id = seq_id;
- art_doc.score = score;
- art_doc.offsets_len = 0;
- art_doc.offsets = nullptr;
-
- art_insert(t, key, KEY_LEN, &art_doc, num_hits);
-}
-
-
-void Collection::index_string_field(const std::string & text, const uint32_t score, art_tree *t,
- uint32_t seq_id, const bool verbatim) const {
- std::vector<std::string> tokens;
- std::unordered_map<std::string, std::vector<uint32_t>> token_to_offsets;
-
- if(verbatim) {
- tokens.push_back(text);
- token_to_offsets[text].push_back(0);
- } else {
- StringUtils::split(text, tokens, " ");
- for(uint32_t i=0; i<tokens.size(); i++) {
- auto & token = tokens[i];
- transform(token.begin(), token.end(), token.begin(), tolower);
- token_to_offsets[token].push_back(i);
- }
- }
-
- for(auto & kv: token_to_offsets) {
- art_document art_doc;
- art_doc.id = seq_id;
- art_doc.score = score;
- art_doc.offsets_len = (uint32_t) kv.second.size();
- art_doc.offsets = new uint32_t[kv.second.size()];
-
- uint32_t num_hits = 0;
-
- const unsigned char *key = (const unsigned char *) kv.first.c_str();
- int key_len = (int) kv.first.length() + 1; // for the terminating \0 char
-
- art_leaf* leaf = (art_leaf *) art_search(t, key, key_len);
- if(leaf != NULL) {
- num_hits = leaf->values->ids.getLength();
- }
-
- num_hits += 1;
-
- for(auto i=0; i<kv.second.size(); i++) {
- art_doc.offsets[i] = kv.second[i];
- }
-
- art_insert(t, key, key_len, &art_doc, num_hits);
- delete [] art_doc.offsets;
- art_doc.offsets = nullptr;
- }
-}
-
-void Collection::index_string_array_field(const std::vector<std::string> & strings, const uint32_t score, art_tree *t,
- uint32_t seq_id, const bool verbatim) const {
- for(const std::string & str: strings) {
- index_string_field(str, score, t, seq_id, verbatim);
- }
-}
-
-void Collection::index_int32_array_field(const std::vector<int32_t> & values, const uint32_t score, art_tree *t,
- uint32_t seq_id) const {
- for(const int32_t value: values) {
- index_int32_field(value, score, t, seq_id);
- }
-}
-
-void Collection::index_int64_array_field(const std::vector<int64_t> & values, const uint32_t score, art_tree *t,
- uint32_t seq_id) const {
- for(const int64_t value: values) {
- index_int64_field(value, score, t, seq_id);
- }
-}
-
-void Collection::index_float_array_field(const std::vector<float> & values, const uint32_t score, art_tree *t,
- uint32_t seq_id) const {
- for(const float value: values) {
- index_float_field(value, score, t, seq_id);
- }
-}
-
-void Collection::do_facets(std::vector<facet> & facets, uint32_t* result_ids, size_t results_size) {
- for(auto & a_facet: facets) {
- // assumed that facet fields have already been validated upstream
- const field & facet_field = facet_schema.at(a_facet.field_name);
- const facet_value & fvalue = facet_index.at(facet_field.name);
-
- for(auto i = 0; i < results_size; i++) {
- uint32_t doc_seq_id = result_ids[i];
- if(fvalue.doc_values.count(doc_seq_id) != 0) {
- // for every result document, get the values associated and increment counter
- const std::vector<uint32_t> & value_indices = fvalue.doc_values.at(doc_seq_id);
- for(auto j = 0; j < value_indices.size(); j++) {
- const std::string & facet_value = fvalue.index_value.at(value_indices.at(j));
- a_facet.result_map[facet_value] += 1;
- }
- }
- }
- }
-}
-
-void Collection::search_candidates(uint32_t* filter_ids, size_t filter_ids_length, std::vector<facet> & facets,
- const std::vector<sort_by> & sort_fields, int & candidate_rank,
- std::vector<std::vector<art_leaf*>> & token_to_candidates,
- std::vector<std::vector<art_leaf*>> & searched_queries, Topster<100> & topster,
- size_t & total_results, uint32_t** all_result_ids, size_t & all_result_ids_len,
- const size_t & max_results, const bool prefix) {
- const size_t combination_limit = 10;
- const size_t prefix_candidates_limit = 5;
-
- auto product = []( long long a, std::vector<art_leaf*>& b ) { return a*b.size(); };
- long long int N = std::accumulate(token_to_candidates.begin(), token_to_candidates.end(), 1LL, product);
-
- for(long long n=0; n<N && n<combination_limit; ++n) {
- // every element in `query_suggestion` contains a token and its associated hits
- std::vector<art_leaf *> query_suggestion = next_suggestion(token_to_candidates, n);
-
- /*for(auto i=0; i < query_suggestion.size(); i++) {
- std::cout << "i: " << i << " - " << query_suggestion[i]->key << std::endl;
- }*/
-
- // initialize results with the starting element (for further intersection)
- size_t result_size = query_suggestion[0]->values->ids.getLength();
- if(result_size == 0) {
- continue;
- }
-
- uint32_t* result_ids = query_suggestion[0]->values->ids.uncompress();
- candidate_rank += 1;
-
- // intersect the document ids for each token to find docs that contain all the tokens (stored in `result_ids`)
- for(auto i=1; i < query_suggestion.size(); i++) {
- uint32_t* out = nullptr;
- uint32_t* ids = query_suggestion[i]->values->ids.uncompress();
- result_size = ArrayUtils::and_scalar(ids, query_suggestion[i]->values->ids.getLength(), result_ids, result_size, &out);
- delete[] ids;
- delete[] result_ids;
- result_ids = out;
- }
-
- if(filter_ids != nullptr) {
- // intersect once again with filter ids
- uint32_t* filtered_result_ids = nullptr;
- size_t filtered_results_size = ArrayUtils::and_scalar(filter_ids, filter_ids_length, result_ids,
- result_size, &filtered_result_ids);
-
- uint32_t* new_all_result_ids;
- all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, filtered_result_ids,
- filtered_results_size, &new_all_result_ids);
- delete [] *all_result_ids;
- *all_result_ids = new_all_result_ids;
-
- do_facets(facets, filtered_result_ids, filtered_results_size);
-
- // go through each matching document id and calculate match score
- score_results(sort_fields, searched_queries.size(), candidate_rank, topster, query_suggestion,
- filtered_result_ids, filtered_results_size);
-
- delete[] filtered_result_ids;
- delete[] result_ids;
- } else {
- do_facets(facets, result_ids, result_size);
-
- uint32_t* new_all_result_ids;
- all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, result_ids,
- result_size, &new_all_result_ids);
- delete [] *all_result_ids;
- *all_result_ids = new_all_result_ids;
-
- score_results(sort_fields, searched_queries.size(), candidate_rank, topster, query_suggestion,
- result_ids, result_size);
- delete[] result_ids;
- }
-
- total_results += topster.size;
- searched_queries.push_back(query_suggestion);
-
- if(!prefix && total_results >= max_results) {
- break;
- }
-
- if(prefix && candidate_rank >= prefix_candidates_limit) {
- break;
- }
- }
-}
-
-size_t Collection::union_of_ids(std::vector<std::pair<uint32_t*, size_t>> & result_array_pairs,
- uint32_t **results_out) {
- uint32_t *results = nullptr;
- size_t results_length = 0;
-
- uint32_t *prev_results = nullptr;
- size_t prev_results_length = 0;
-
- for(const std::pair<uint32_t*, size_t> & result_array_pair: result_array_pairs) {
- results_length = ArrayUtils::or_scalar(prev_results, prev_results_length, result_array_pair.first,
- result_array_pair.second, &results);
- delete [] prev_results;
- prev_results = results;
- prev_results_length = results_length;
- }
-
- *results_out = results;
- return results_length;
-}
-
-Option<uint32_t> Collection::do_filtering(uint32_t** filter_ids_out, const std::string & simple_filter_str) {
- // parse the filter string
- std::vector<std::string> filter_blocks;
- StringUtils::split(simple_filter_str, filter_blocks, "&&");
-
- std::vector<filter> filters;
-
- for(const std::string & filter_block: filter_blocks) {
- // split into [field_name, value]
- std::vector<std::string> expression_parts;
- StringUtils::split(filter_block, expression_parts, ":");
- if(expression_parts.size() != 2) {
- return Option<>(400, "Could not parse the filter query.");
- }
-
- const std::string & field_name = expression_parts[0];
- if(search_schema.count(field_name) == 0) {
- return Option<>(400, "Could not find a filter field named `" + field_name + "` in the schema.");
- }
-
- field _field = search_schema.at(field_name);
- std::string & raw_value = expression_parts[1];
- filter f;
-
- if(_field.is_integer() || _field.is_float()) {
- // could be a single value or a list
- if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
- std::vector<std::string> filter_values;
- StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");
-
- for(const std::string & filter_value: filter_values) {
- if(_field.is_integer() && !StringUtils::is_integer(filter_value)) {
- return Option<>(400, "Error with field `" + _field.name + "`: Not an integer.");
- }
-
- if(_field.is_float() && !StringUtils::is_float(filter_value)) {
- return Option<>(400, "Error with field `" + _field.name + "`: Not a float.");
- }
- }
-
- f = {field_name, filter_values, EQUALS};
- } else {
- Option<NUM_COMPARATOR> op_comparator = filter::extract_num_comparator(raw_value);
- if(!op_comparator.ok()) {
- return Option<>(400, "Error with field `" + _field.name + "`: " + op_comparator.error());
- }
-
- // extract numerical value
- std::string filter_value;
- if(op_comparator.get() == LESS_THAN || op_comparator.get() == GREATER_THAN) {
- filter_value = raw_value.substr(1);
- } else if(op_comparator.get() == LESS_THAN_EQUALS || op_comparator.get() == GREATER_THAN_EQUALS) {
- filter_value = raw_value.substr(2);
- } else {
- // EQUALS
- filter_value = raw_value;
- }
-
- filter_value = StringUtils::trim(filter_value);
-
- if(_field.is_integer() && !StringUtils::is_integer(filter_value)) {
- return Option<>(400, "Error with field `" + _field.name + "`: Not an integer.");
- }
-
- if(_field.is_float() && !StringUtils::is_float(filter_value)) {
- return Option<>(400, "Error with field `" + _field.name + "`: Not a float.");
- }
-
- f = {field_name, {filter_value}, op_comparator.get()};
- }
- } else if(_field.is_string()) {
- if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
- std::vector<std::string> filter_values;
- StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");
- f = {field_name, filter_values, EQUALS};
- } else {
- f = {field_name, {raw_value}, EQUALS};
- }
- } else {
- return Option<>(400, "Error with field `" + _field.name + "`: Unidentified field type.");
- }
-
- filters.push_back(f);
- }
-
- uint32_t* filter_ids = nullptr;
- uint32_t filter_ids_length = 0;
-
- // process the filters first
- for(const filter & a_filter: filters) {
- if(search_index.count(a_filter.field_name) != 0) {
- art_tree* t = search_index.at(a_filter.field_name);
- field f = search_schema.at(a_filter.field_name);
- std::vector<std::pair<uint32_t*, size_t>> filter_result_array_pairs;
-
- if(f.is_integer()) {
- std::vector<const art_leaf*> leaves;
-
- for(const std::string & filter_value: a_filter.values) {
- if(f.type == field_types::INT32 || f.type == field_types::INT32_ARRAY) {
- int32_t value = (int32_t) std::stoi(filter_value);
- art_int32_search(t, value, a_filter.compare_operator, leaves);
- } else {
- int64_t value = (int64_t) std::stoi(filter_value);
- art_int64_search(t, value, a_filter.compare_operator, leaves);
- }
-
- for(const art_leaf* leaf: leaves) {
- filter_result_array_pairs.push_back(std::make_pair(leaf->values->ids.uncompress(),
- leaf->values->ids.getLength()));
- }
- }
- } else if(f.is_float()) {
- std::vector<const art_leaf*> leaves;
-
- for(const std::string & filter_value: a_filter.values) {
- float value = (float) std::atof(filter_value.c_str());
- art_float_search(t, value, a_filter.compare_operator, leaves);
- for(const art_leaf* leaf: leaves) {
- filter_result_array_pairs.push_back(std::make_pair(leaf->values->ids.uncompress(),
- leaf->values->ids.getLength()));
- }
- }
- } else if(f.is_string()) {
- for(const std::string & filter_value: a_filter.values) {
- // we have to tokenize the string, standardize it and then do an exact match
- std::vector<std::string> str_tokens;
- StringUtils::split(filter_value, str_tokens, " ");
-
- uint32_t* filtered_ids = nullptr;
- size_t filtered_size = 0;
-
- for(auto i = 0; i < str_tokens.size(); i++) {
- std::string & str_token = str_tokens[i];
- StringUtils::normalize(str_token);
- art_leaf* leaf = (art_leaf *) art_search(t, (const unsigned char*) str_token.c_str(),
- str_token.length()+1);
- if(leaf == nullptr) {
- continue;
- }
-
- if(i == 0) {
- filtered_ids = leaf->values->ids.uncompress();
- filtered_size = leaf->values->ids.getLength();
- } else {
- // do AND for an exact match
- uint32_t* out = nullptr;
- uint32_t* leaf_ids = leaf->values->ids.uncompress();
- filtered_size = ArrayUtils::and_scalar(filtered_ids, filtered_size, leaf_ids,
- leaf->values->ids.getLength(), &out);
- delete[] leaf_ids;
- delete[] filtered_ids;
- filtered_ids = out;
- }
- }
-
- filter_result_array_pairs.push_back(std::make_pair(filtered_ids, filtered_size));
- }
- }
-
- uint32_t* result_ids = nullptr;
- size_t result_ids_length = union_of_ids(filter_result_array_pairs, &result_ids);
-
- if(filter_ids == nullptr) {
- filter_ids = result_ids;
- filter_ids_length = result_ids_length;
- } else {
- uint32_t* filtered_results = nullptr;
- filter_ids_length = ArrayUtils::and_scalar(filter_ids, filter_ids_length, result_ids,
- result_ids_length, &filtered_results);
- delete [] result_ids;
- delete [] filter_ids;
- filter_ids = filtered_results;
- }
-
- for(std::pair<uint32_t*, size_t> & filter_result_array_pair: filter_result_array_pairs) {
- delete[] filter_result_array_pair.first;
- }
- }
- }
-
- *filter_ids_out = filter_ids;
- return Option<>(filter_ids_length);
-}
-
Option<nlohmann::json> Collection::search(std::string query, const std::vector<std::string> search_fields,
const std::string & simple_filter_query, const std::vector<std::string> & facet_fields,
const std::vector<sort_by> & sort_fields, const int num_typos,
const size_t per_page, const size_t page,
const token_ordering token_order, const bool prefix) {
- nlohmann::json result = nlohmann::json::object();
std::vector<facet> facets;
// validate search fields
@@ -785,7 +262,7 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
std::vector<sort_by> sort_fields_std;
for(const sort_by & _sort_field: sort_fields) {
- if(sort_index.count(_sort_field.name) == 0) {
+ if(sort_schema.count(_sort_field.name) == 0) {
std::string error = "Could not find a field named `" + _sort_field.name + "` in the schema for sorting.";
return Option<nlohmann::json>(400, error);
}
@@ -812,45 +289,18 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
return Option<nlohmann::json>(422, message);
}
- const size_t num_results = (page * per_page);
-
- // process the filters
- uint32_t* filter_ids = nullptr;
- Option<uint32_t> op_filter_ids_length = do_filtering(&filter_ids, simple_filter_query);
- if(!op_filter_ids_length.ok()) {
- return Option<nlohmann::json>(op_filter_ids_length.code(), op_filter_ids_length.error());
- }
-
- const uint32_t filter_ids_length = op_filter_ids_length.get();
-
- // Order of `fields` are used to sort results
auto begin = std::chrono::high_resolution_clock::now();
- std::vector<std::pair<int, Topster<100>::KV>> field_order_kvs;
- uint32_t* all_result_ids = nullptr;
- size_t all_result_ids_len = 0;
// all search queries that were used for generating the results
std::vector<std::vector<art_leaf*>> searched_queries;
- int searched_queries_index = 0;
-
- for(int i = 0; i < search_fields.size(); i++) {
- Topster<100> topster;
- const std::string & field = search_fields[i];
- // proceed to query search only when no filters are provided or when filtering produces results
- if(simple_filter_query.size() == 0 || filter_ids_length > 0) {
- search_field(query, field, filter_ids, filter_ids_length, facets, sort_fields_std, num_typos, num_results,
- searched_queries, searched_queries_index, topster, &all_result_ids, all_result_ids_len, token_order, prefix);
- topster.sort();
- }
- // order of fields specified matter: matching docs from earlier fields are more important
- for(auto t = 0; t < topster.size && t < num_results; t++) {
- field_order_kvs.push_back(std::make_pair(search_fields.size() - i, topster.getKV(t)));
- }
- }
+ std::vector<std::pair<int, Topster<100>::KV>> field_order_kvs;
+ size_t all_result_ids_len = 0;
- delete [] filter_ids;
- delete [] all_result_ids;
+ for(Index* index: indices) {
+ index->search(query, search_fields, simple_filter_query, facets, sort_fields_std, num_typos,
+ per_page, page, token_order, prefix, field_order_kvs, all_result_ids_len, searched_queries);
+ }
// All fields are sorted descending
std::sort(field_order_kvs.begin(), field_order_kvs.end(),
@@ -859,6 +309,8 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
std::tie(b.second.match_score, b.second.primary_attr, b.second.secondary_attr, b.first, b.second.key);
});
+ nlohmann::json result = nlohmann::json::object();
+
result["hits"] = nlohmann::json::array();
result["found"] = all_result_ids_len;
@@ -994,359 +446,6 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
return result;
}
-/*
- 1. Split the query into tokens
- 2. Outer loop will generate bounded cartesian product with costs for each token
- 3. Inner loop will iterate on each token with associated cost
- 4. Cartesian product of the results of the token searches will be used to form search phrases
- (cartesian product adapted from: http://stackoverflow.com/a/31169617/131050)
- 4. Intersect the lists to find docs that match each phrase
- 5. Sort the docs based on some ranking criteria
-*/
-void Collection::search_field(std::string & query, const std::string & field, uint32_t *filter_ids, size_t filter_ids_length,
- std::vector<facet> & facets, const std::vector<sort_by> & sort_fields, const int num_typos,
- const size_t num_results, std::vector<std::vector<art_leaf*>> & searched_queries,
- int & searched_queries_index, Topster<100> &topster, uint32_t** all_result_ids, size_t & all_result_ids_len,
- const token_ordering token_order, const bool prefix) {
- std::vector<std::string> tokens;
- StringUtils::split(query, tokens, " ");
-
- const int max_cost = (num_typos < 0 || num_typos > 2) ? 2 : num_typos;
- const size_t max_results = std::min(num_results, (size_t) Collection::MAX_RESULTS);
-
- size_t total_results = topster.size;
-
- // To prevent us from doing ART search repeatedly as we iterate through possible corrections
- spp::sparse_hash_map<std::string, std::vector<art_leaf*>> token_cost_cache;
-
- // Used to drop the least occurring token(s) for partial searches
- spp::sparse_hash_map<std::string, uint32_t> token_to_count;
-
- std::vector<std::vector<int>> token_to_costs;
-
- for(size_t token_index = 0; token_index < tokens.size(); token_index++) {
- std::vector<int> all_costs;
- const size_t token_len = tokens[token_index].length();
-
- // This ensures that we don't end up doing a cost of 1 for a single char etc.
- int bounded_cost = max_cost;
- if(token_len > 0 && max_cost >= token_len && (token_len == 1 || token_len == 2)) {
- bounded_cost = token_len - 1;
- }
-
- for(int cost = 0; cost <= bounded_cost; cost++) {
- all_costs.push_back(cost);
- }
-
- token_to_costs.push_back(all_costs);
- StringUtils::normalize(tokens[token_index]);
- }
-
- // stores candidates for each token, i.e. i-th index would have all possible tokens with a cost of "c"
- std::vector<std::vector<art_leaf*>> token_to_candidates;
-
- const size_t combination_limit = 10;
- auto product = []( long long a, std::vector<int>& b ) { return a*b.size(); };
- int candidate_rank = 0;
- long long n = 0;
- long long int N = std::accumulate(token_to_costs.begin(), token_to_costs.end(), 1LL, product);
-
- while(n < N && n < combination_limit) {
- // Outerloop generates combinations of [cost to max_cost] for each token
- // For e.g. for a 3-token query: [0, 0, 0], [0, 0, 1], [0, 1, 1] etc.
- std::vector<uint32_t> costs(token_to_costs.size());
- ldiv_t q { n, 0 };
- for(long long i = (token_to_costs.size() - 1); 0 <= i ; --i ) {
- q = ldiv(q.quot, token_to_costs[i].size());
- costs[i] = token_to_costs[i][q.rem];
- }
-
- token_to_candidates.clear();
- int token_index = 0;
-
- while(token_index < tokens.size()) {
- // For each token, look up the generated cost for this iteration and search using that cost
- std::string token = tokens[token_index];
- const std::string token_cost_hash = token + std::to_string(costs[token_index]);
-
- std::vector<art_leaf*> leaves;
- /*std::cout << "\nSearching for: " << token << " - cost: " << costs[token_index] << ", candidate_rank: "
- << candidate_rank << std::endl;*/
-
- if(token_cost_cache.count(token_cost_hash) != 0) {
- leaves = token_cost_cache[token_cost_hash];
- } else {
- // prefix should apply only for last token
- const bool prefix_search = prefix && ((token_index == tokens.size()-1) ? true : false);
- const int token_len = prefix_search ? (int) token.length() : (int) token.length() + 1;
- const int max_candidates = prefix_search ? 5 : 3;
-
- art_fuzzy_search(search_index.at(field), (const unsigned char *) token.c_str(), token_len,
- costs[token_index], costs[token_index], max_candidates, token_order, prefix_search, leaves);
-
- if(!leaves.empty()) {
- token_cost_cache.emplace(token_cost_hash, leaves);
- }
- }
-
- if(!leaves.empty()) {
- //!log_leaves(costs[token_index], token, leaves);
- token_to_candidates.push_back(leaves);
- token_to_count[token] = std::max(token_to_count[token], leaves.at(0)->values->ids.getLength());
- } else {
- // No result at `cost = costs[token_index]`. Remove costs until `cost` for token and re-do combinations
- auto it = std::find(token_to_costs[token_index].begin(), token_to_costs[token_index].end(), costs[token_index]);
- if(it != token_to_costs[token_index].end()) {
- token_to_costs[token_index].erase(it);
-
- // no more costs left for this token, clean up
- if(token_to_costs[token_index].empty()) {
- token_to_costs.erase(token_to_costs.begin()+token_index);
- tokens.erase(tokens.begin()+token_index);
- token_index--;
- }
- }
-
- // To continue outerloop on new cost combination
- n = -1;
- N = std::accumulate(token_to_costs.begin(), token_to_costs.end(), 1LL, product);
- break;
- }
-
- token_index++;
- }
-
- if(token_to_candidates.size() != 0 && token_to_candidates.size() == tokens.size()) {
- // If all tokens were found, go ahead and search for candidates with what we have so far
- search_candidates(filter_ids, filter_ids_length, facets, sort_fields, candidate_rank, token_to_candidates,
- searched_queries, topster, total_results, all_result_ids, all_result_ids_len,
- max_results, prefix);
-
- if (!prefix && total_results >= max_results) {
- // If we don't find enough results, we continue outerloop (looking at tokens with greater cost)
- break;
- }
-
- // only allow upto 10 prefix candidate tokens
- if(prefix && candidate_rank > 10) {
- break;
- }
- }
-
- n++;
- }
-
- // When there are not enough overall results and atleast one token has results
- if(topster.size < max_results && token_to_count.size() > 1) {
- // Drop token with least hits and try searching again
- std::string truncated_query;
-
- std::vector<std::pair<std::string, uint32_t>> token_count_pairs;
- for (auto itr = token_to_count.begin(); itr != token_to_count.end(); ++itr) {
- token_count_pairs.push_back(*itr);
- }
-
- std::sort(token_count_pairs.begin(), token_count_pairs.end(), [=]
- (const std::pair<std::string, uint32_t>& a, const std::pair<std::string, uint32_t>& b) {
- return a.second > b.second;
- }
- );
-
- for(uint32_t i = 0; i < token_count_pairs.size()-1; i++) {
- // iterate till last but one
- truncated_query += " " + token_count_pairs.at(i).first;
- }
-
- return search_field(truncated_query, field, filter_ids, filter_ids_length, facets, sort_fields, num_typos,
- num_results, searched_queries, candidate_rank, topster, all_result_ids, all_result_ids_len,
- token_order, prefix);
- }
-}
-
-void Collection::log_leaves(const int cost, const std::string &token, const std::vector<art_leaf *> &leaves) const {
- printf("Token: %s, cost: %d, candidates: \n", token.c_str(), cost);
- for(auto i=0; i < leaves.size(); i++) {
- printf("%.*s, ", leaves[i]->key_len, leaves[i]->key);
- printf("frequency: %d, max_score: %d\n", leaves[i]->values->ids.getLength(), leaves[i]->max_score);
- /*for(auto j=0; j<leaves[i]->values->ids.getLength(); j++) {
- printf("id: %d\n", leaves[i]->values->ids.at(j));
- }*/
- }
-}
-
-void Collection::score_results(const std::vector<sort_by> & sort_fields, const int & query_index, const int & candidate_rank,
- Topster<100> & topster, const std::vector<art_leaf *> &query_suggestion,
- const uint32_t *result_ids, const size_t result_size) const {
-
- const int max_candidate_rank = 250;
- spp::sparse_hash_map<const art_leaf*, uint32_t*> leaf_to_indices;
-
- for (art_leaf *token_leaf : query_suggestion) {
- uint32_t *indices = new uint32_t[result_size];
- token_leaf->values->ids.indexOf(result_ids, result_size, indices);
- leaf_to_indices.emplace(token_leaf, indices);
- }
-
- spp::sparse_hash_map<uint32_t, number_t> * primary_rank_scores = nullptr;
- spp::sparse_hash_map<uint32_t, number_t> * secondary_rank_scores = nullptr;
-
- // Used for asc/desc ordering. NOTE: Topster keeps biggest keys (i.e. it's desc in nature)
- number_t primary_rank_factor;
- number_t secondary_rank_factor;
-
- if(sort_fields.size() > 0) {
- // assumed that rank field exists in the index - checked earlier in the chain
- primary_rank_scores = sort_index.at(sort_fields[0].name);
-
- // initialize primary_rank_factor
- field sort_field = sort_schema.at(sort_fields[0].name);
- if(sort_field.is_single_integer()) {
- primary_rank_factor = ((int64_t) 1);
- } else {
- primary_rank_factor = ((float) 1);
- }
-
- if(sort_fields[0].order == sort_field_const::asc) {
- primary_rank_factor = -primary_rank_factor;
- }
- }
-
- if(sort_fields.size() > 1) {
- secondary_rank_scores = sort_index.at(sort_fields[1].name);
-
- // initialize secondary_rank_factor
- field sort_field = sort_schema.at(sort_fields[1].name);
- if(sort_field.is_single_integer()) {
- secondary_rank_factor = ((int64_t) 1);
- } else {
- secondary_rank_factor = ((float) 1);
- }
-
- if(sort_fields[1].order == sort_field_const::asc) {
- secondary_rank_factor = -secondary_rank_factor;
- }
- }
-
- for(auto i=0; i<result_size; i++) {
- uint32_t seq_id = result_ids[i];
- MatchScore mscore;
-
- if(query_suggestion.size() == 1) {
- // short circuit to speed up single token searches (use dummy offsets for now)
- char offset_diffs[16];
- std::fill_n(offset_diffs, 16, 0);
- mscore = MatchScore(1, 0, 0, offset_diffs);
- } else {
- std::vector<std::vector<uint16_t>> token_positions;
- populate_token_positions(query_suggestion, leaf_to_indices, i, token_positions);
- mscore = MatchScore::match_score(seq_id, token_positions);
- }
-
- int candidate_rank_score = max_candidate_rank - candidate_rank;
-
- // Construct a single match_score from individual components (for multi-field sort)
- const uint64_t match_score = ((uint64_t)(mscore.words_present) << 16) +
- (candidate_rank_score << 8) +
- (MAX_SEARCH_TOKENS - mscore.distance);
-
- const int64_t default_score = 0;
- const number_t & primary_rank_score = (primary_rank_scores && primary_rank_scores->count(seq_id) > 0) ?
- primary_rank_scores->at(seq_id) : default_score;
- const number_t & secondary_rank_score = (secondary_rank_scores && secondary_rank_scores->count(seq_id) > 0) ?
- secondary_rank_scores->at(seq_id) : default_score;
-
- const number_t & primary_rank_value = primary_rank_score * primary_rank_factor;
- const number_t & secondary_rank_value = secondary_rank_score * secondary_rank_factor;
- topster.add(seq_id, query_index, match_score, primary_rank_value, secondary_rank_value);
-
- /*std::cout << "candidate_rank: " << candidate_rank << ", candidate_rank_score: " << candidate_rank_score
- << ", words_present: " << mscore.words_present << ", match_score: " << match_score
- << ", primary_rank_score: " << primary_rank_score.intval << ", seq_id: " << seq_id << std::endl;*/
- }
-
- for (auto it = leaf_to_indices.begin(); it != leaf_to_indices.end(); it++) {
- delete [] it->second;
- it->second = nullptr;
- }
-}
-
-void Collection::populate_token_positions(const std::vector<art_leaf *> &query_suggestion,
- spp::sparse_hash_map<const art_leaf *, uint32_t *> &leaf_to_indices,
- size_t result_index, std::vector<std::vector<uint16_t>> &token_positions) const {
- // for each token in the query, find the positions that it appears in this document
- for (const art_leaf *token_leaf : query_suggestion) {
- std::vector<uint16_t> positions;
- int doc_index = leaf_to_indices.at(token_leaf)[result_index];
- if(doc_index == token_leaf->values->ids.getLength()) {
- continue;
- }
-
- uint32_t start_offset = token_leaf->values->offset_index.at(doc_index);
- uint32_t end_offset = (doc_index == token_leaf->values->ids.getLength() - 1) ?
- token_leaf->values->offsets.getLength() :
- token_leaf->values->offset_index.at(doc_index+1);
-
- while(start_offset < end_offset) {
- positions.push_back((uint16_t) token_leaf->values->offsets.at(start_offset));
- start_offset++;
- }
-
- token_positions.push_back(positions);
- }
-}
-
-inline std::vector<art_leaf *> Collection::next_suggestion(const std::vector<std::vector<art_leaf *>> &token_leaves,
- long long int n) {
- std::vector<art_leaf*> query_suggestion(token_leaves.size());
-
- // generate the next combination from `token_leaves` and store it in `query_suggestion`
- ldiv_t q { n, 0 };
- for(long long i = 0 ; i < token_leaves.size(); i++) {
- q = ldiv(q.quot, token_leaves[i].size());
- query_suggestion[i] = token_leaves[i][q.rem];
- }
-
- // sort ascending based on matched documents for each token for faster intersection
- sort(query_suggestion.begin(), query_suggestion.end(), [](const art_leaf* left, const art_leaf* right) {
- return left->values->ids.getLength() < right->values->ids.getLength();
- });
-
- return query_suggestion;
-}
-
-void Collection::remove_and_shift_offset_index(sorted_array &offset_index, const uint32_t *indices_sorted,
- const uint32_t indices_length) {
- uint32_t *curr_array = offset_index.uncompress();
- uint32_t *new_array = new uint32_t[offset_index.getLength()];
-
- new_array[0] = 0;
- uint32_t new_index = 0;
- uint32_t curr_index = 0;
- uint32_t indices_counter = 0;
- uint32_t shift_value = 0;
-
- while(curr_index < offset_index.getLength()) {
- if(indices_counter < indices_length && curr_index >= indices_sorted[indices_counter]) {
- // skip copying
- if(curr_index == indices_sorted[indices_counter]) {
- curr_index++;
- const uint32_t diff = curr_index == offset_index.getLength() ?
- 0 : (offset_index.at(curr_index) - offset_index.at(curr_index-1));
-
- shift_value += diff;
- }
- indices_counter++;
- } else {
- new_array[new_index++] = curr_array[curr_index++] - shift_value;
- }
- }
-
- offset_index.load(new_array, new_index);
-
- delete[] curr_array;
- delete[] new_array;
-}
-
Option<nlohmann::json> Collection::get(const std::string & id) {
std::string seq_id_str;
StoreStatus status = store->get(get_doc_id_key(id), seq_id_str);
@@ -1390,114 +489,8 @@ Option<std::string> Collection::remove(const std::string & id, const bool remove
return Option<std::string>(500, "Error while parsing stored document.");
}
- for(auto & name_field: search_schema) {
- // Go through all the field names and find the keys+values so that they can be removed from in-memory index
- std::vector<std::string> tokens;
- if(name_field.second.type == field_types::STRING) {
- StringUtils::split(document[name_field.first], tokens, " ");
- } else if(name_field.second.type == field_types::STRING_ARRAY) {
- tokens = document[name_field.first].get<std::vector<std::string>>();
- } else if(name_field.second.type == field_types::INT32) {
- const int KEY_LEN = 8;
- unsigned char key[KEY_LEN];
- int32_t value = document[name_field.first].get<int32_t>();
- encode_int32(value, key);
- tokens.push_back(std::string((char*)key, KEY_LEN));
- } else if(name_field.second.type == field_types::INT32_ARRAY) {
- std::vector<int32_t> values = document[name_field.first].get<std::vector<int32_t>>();
- for(const int32_t value: values) {
- const int KEY_LEN = 8;
- unsigned char key[KEY_LEN];
- encode_int32(value, key);
- tokens.push_back(std::string((char*)key, KEY_LEN));
- }
- } else if(name_field.second.type == field_types::INT64) {
- const int KEY_LEN = 8;
- unsigned char key[KEY_LEN];
- int64_t value = document[name_field.first].get<int64_t>();
- encode_int64(value, key);
- tokens.push_back(std::string((char*)key, KEY_LEN));
- } else if(name_field.second.type == field_types::INT64_ARRAY) {
- std::vector<int64_t> values = document[name_field.first].get<std::vector<int64_t>>();
- for(const int64_t value: values) {
- const int KEY_LEN = 8;
- unsigned char key[KEY_LEN];
- encode_int64(value, key);
- tokens.push_back(std::string((char*)key, KEY_LEN));
- }
- } else if(name_field.second.type == field_types::FLOAT) {
- const int KEY_LEN = 8;
- unsigned char key[KEY_LEN];
- int64_t value = document[name_field.first].get<int64_t>();
- encode_float(value, key);
- tokens.push_back(std::string((char*)key, KEY_LEN));
- } else if(name_field.second.type == field_types::FLOAT_ARRAY) {
- std::vector<float> values = document[name_field.first].get<std::vector<float>>();
- for(const float value: values) {
- const int KEY_LEN = 8;
- unsigned char key[KEY_LEN];
- encode_float(value, key);
- tokens.push_back(std::string((char*)key, KEY_LEN));
- }
- }
-
- for(auto & token: tokens) {
- const unsigned char *key;
- int key_len;
-
- if(name_field.second.type == field_types::STRING_ARRAY || name_field.second.type == field_types::STRING) {
- StringUtils::normalize(token);
- key = (const unsigned char *) token.c_str();
- key_len = (int) (token.length() + 1);
- } else {
- key = (const unsigned char *) token.c_str();
- key_len = (int) (token.length());
- }
-
- art_leaf* leaf = (art_leaf *) art_search(search_index.at(name_field.first), key, key_len);
- if(leaf != NULL) {
- uint32_t seq_id_values[1] = {seq_id};
- uint32_t doc_index = leaf->values->ids.indexOf(seq_id);
-
- if(doc_index == leaf->values->ids.getLength()) {
- // not found - happens when 2 tokens repeat in a field, e.g "is it or is is not?"
- continue;
- }
-
- uint32_t start_offset = leaf->values->offset_index.at(doc_index);
- uint32_t end_offset = (doc_index == leaf->values->ids.getLength() - 1) ?
- leaf->values->offsets.getLength() :
- leaf->values->offset_index.at(doc_index+1);
-
- uint32_t doc_indices[1] = {doc_index};
- remove_and_shift_offset_index(leaf->values->offset_index, doc_indices, 1);
-
- leaf->values->offsets.remove_index(start_offset, end_offset);
- leaf->values->ids.remove_values(seq_id_values, 1);
-
- /*len = leaf->values->offset_index.getLength();
- for(auto i=0; i<len; i++) {
- std::cout << "i: " << i << ", val: " << leaf->values->offset_index.at(i) << std::endl;
- }
- std::cout << "----" << std::endl;*/
-
- if(leaf->values->ids.getLength() == 0) {
- art_values* values = (art_values*) art_delete(search_index.at(name_field.first), key, key_len);
- delete values;
- values = nullptr;
- }
- }
- }
- }
-
- // remove facets if any
- for(auto & field_facet_value: facet_index) {
- field_facet_value.second.doc_values.erase(seq_id);
- }
-
- // remove sort index if any
- for(auto & field_doc_value_map: sort_index) {
- field_doc_value_map.second->erase(seq_id);
+ for(Index* index: indices) {
+ index->remove(seq_id, document);
}
if(remove_from_store) {
diff --git a/src/index.cpp b/src/index.cpp
index 2c548e5b..a17afbda 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -7,34 +7,30 @@
#include <string_utils.h>
#include <art.h>
-Collection::Collection(const std::string name, const uint32_t collection_id, const uint32_t next_seq_id, Store *store,
- const std::vector<field> &fields, const std::string & token_ranking_field):
- name(name), collection_id(collection_id), next_seq_id(next_seq_id), store(store),
- token_ranking_field(token_ranking_field) {
+Index::Index(const std::string name, spp::sparse_hash_map<std::string, field> search_schema,
+ spp::sparse_hash_map<std::string, field> facet_schema, spp::sparse_hash_map<std::string, field> sort_schema):
+ name(name), search_schema(search_schema), facet_schema(facet_schema), sort_schema(sort_schema) {
- for(const field& field: fields) {
+ for(const auto pair: search_schema) {
art_tree *t = new art_tree;
art_tree_init(t);
- search_index.emplace(field.name, t);
- search_schema.emplace(field.name, field);
+ search_index.emplace(pair.first, t);
+ }
- if(field.is_facet()) {
- facet_value fvalue;
- facet_index.emplace(field.name, fvalue);
- facet_schema.emplace(field.name, field);
- }
+ for(const auto pair: facet_schema) {
+ facet_value fvalue;
+ facet_index.emplace(pair.first, fvalue);
+ }
- if(field.is_single_integer() || field.is_single_float()) {
- spp::sparse_hash_map<uint32_t, number_t> * doc_to_score = new spp::sparse_hash_map<uint32_t, number_t>();
- sort_index.emplace(field.name, doc_to_score);
- sort_schema.emplace(field.name, field);
- }
+ for(const auto pair: sort_schema) {
+ spp::sparse_hash_map<uint32_t, number_t> * doc_to_score = new spp::sparse_hash_map<uint32_t, number_t>();
+ sort_index.emplace(pair.first, doc_to_score);
}
num_documents = 0;
}
-Collection::~Collection() {
+Index::~Index() {
for(auto & name_tree: search_index) {
art_tree_destroy(name_tree.second);
delete name_tree.second;
@@ -51,185 +47,8 @@ Collection::~Collection() {
sort_index.clear();
}
-uint32_t Collection::get_next_seq_id() {
- store->increment(get_next_seq_id_key(name), 1);
- return next_seq_id++;
-}
-
-void Collection::set_next_seq_id(uint32_t seq_id) {
- next_seq_id = seq_id;
-}
-
-void Collection::increment_next_seq_id_field() {
- next_seq_id++;
-}
-
-Option<std::string> Collection::add(const std::string & json_str) {
- nlohmann::json document;
- try {
- document = nlohmann::json::parse(json_str);
- } catch(...) {
- return Option<std::string>(400, "Bad JSON.");
- }
-
- uint32_t seq_id = get_next_seq_id();
- std::string seq_id_str = std::to_string(seq_id);
-
- if(document.count("id") == 0) {
- document["id"] = seq_id_str;
- } else if(!document["id"].is_string()) {
- return Option<std::string>(400, "Document's `id` field should be a string.");
- }
-
- std::string doc_id = document["id"];
-
- const Option<uint32_t> & index_memory_op = index_in_memory(document, seq_id);
-
- if(!index_memory_op.ok()) {
- return Option<std::string>(index_memory_op.code(), index_memory_op.error());
- }
-
- store->insert(get_doc_id_key(document["id"]), seq_id_str);
- store->insert(get_seq_id_key(seq_id), document.dump());
-
- return Option<std::string>(doc_id);
-}
-
-Option<uint32_t> Collection::validate_index_in_memory(const nlohmann::json &document, uint32_t seq_id) {
- if(!token_ranking_field.empty() && document.count(token_ranking_field) == 0) {
- return Option<>(400, "Field `" + token_ranking_field + "` has been declared as a token ranking field, "
- "but is not found in the document.");
- }
-
- if(!token_ranking_field.empty() && !document[token_ranking_field].is_number_integer() &&
- !document[token_ranking_field].is_number_float()) {
- return Option<>(400, "Token ranking field `" + token_ranking_field + "` must be a number.");
- }
-
- if(!token_ranking_field.empty() && document[token_ranking_field].is_number_integer() &&
- document[token_ranking_field].get<int64_t>() > std::numeric_limits<int32_t>::max()) {
- return Option<>(400, "Token ranking field `" + token_ranking_field + "` exceeds maximum value of int32.");
- }
-
- if(!token_ranking_field.empty() && document[token_ranking_field].is_number_float() &&
- document[token_ranking_field].get<float>() > std::numeric_limits<float>::max()) {
- return Option<>(400, "Token ranking field `" + token_ranking_field + "` exceeds maximum value of a float.");
- }
-
- for(const std::pair<std::string, field> & field_pair: search_schema) {
- const std::string & field_name = field_pair.first;
-
- if(document.count(field_name) == 0) {
- return Option<>(400, "Field `" + field_name + "` has been declared in the schema, "
- "but is not found in the document.");
- }
-
- art_tree *t = search_index.at(field_name);
-
- if(field_pair.second.type == field_types::STRING) {
- if(!document[field_name].is_string()) {
- return Option<>(400, "Field `" + field_name + "` must be a string.");
- }
- } else if(field_pair.second.type == field_types::INT32) {
- if(!document[field_name].is_number_integer()) {
- return Option<>(400, "Field `" + field_name + "` must be an int32.");
- }
-
- if(document[field_name].get<int64_t>() > INT32_MAX) {
- return Option<>(400, "Field `" + field_name + "` exceeds maximum value of int32.");
- }
- } else if(field_pair.second.type == field_types::INT64) {
- if(!document[field_name].is_number_integer()) {
- return Option<>(400, "Field `" + field_name + "` must be an int64.");
- }
- } else if(field_pair.second.type == field_types::FLOAT) {
- if(!document[field_name].is_number()) { // allows integer to be passed to a float field
- return Option<>(400, "Field `" + field_name + "` must be a float.");
- }
- } else if(field_pair.second.type == field_types::STRING_ARRAY) {
- if(!document[field_name].is_array()) {
- return Option<>(400, "Field `" + field_name + "` must be a string array.");
- }
- if(document[field_name].size() > 0 && !document[field_name][0].is_string()) {
- return Option<>(400, "Field `" + field_name + "` must be a string array.");
- }
- } else if(field_pair.second.type == field_types::INT32_ARRAY) {
- if(!document[field_name].is_array()) {
- return Option<>(400, "Field `" + field_name + "` must be an int32 array.");
- }
-
- if(document[field_name].size() > 0 && !document[field_name][0].is_number_integer()) {
- return Option<>(400, "Field `" + field_name + "` must be an int32 array.");
- }
- } else if(field_pair.second.type == field_types::INT64_ARRAY) {
- if(!document[field_name].is_array()) {
- return Option<>(400, "Field `" + field_name + "` must be an int64 array.");
- }
-
- if(document[field_name].size() > 0 && !document[field_name][0].is_number_integer()) {
- return Option<>(400, "Field `" + field_name + "` must be an int64 array.");
- }
- } else if(field_pair.second.type == field_types::FLOAT_ARRAY) {
- if(!document[field_name].is_array()) {
- return Option<>(400, "Field `" + field_name + "` must be a float array.");
- }
-
- if(document[field_name].size() > 0 && !document[field_name][0].is_number_float()) {
- return Option<>(400, "Field `" + field_name + "` must be a float array.");
- }
- }
- }
-
- for(const std::pair<std::string, field> & field_pair: facet_schema) {
- const std::string & field_name = field_pair.first;
-
- if(document.count(field_name) == 0) {
- return Option<>(400, "Field `" + field_name + "` has been declared as a facet field in the schema, "
- "but is not found in the document.");
- }
-
- facet_value & fvalue = facet_index.at(field_name);
- if(field_pair.second.type == field_types::STRING) {
- if(!document[field_name].is_string()) {
- return Option<>(400, "Facet field `" + field_name + "` must be a string.");
- }
- } else if(field_pair.second.type == field_types::STRING_ARRAY) {
- if(!document[field_name].is_array()) {
- return Option<>(400, "Facet field `" + field_name + "` must be a string array.");
- }
-
- if(document[field_name].size() > 0 && !document[field_name][0].is_string()) {
- return Option<>(400, "Facet field `" + field_name + "` must be a string array.");
- }
- } else {
- return Option<>(400, "Facet field `" + field_name + "` must be a string or a string[].");
- }
- }
-
- return Option<>(200);
-}
-
-Option<uint32_t> Collection::index_in_memory(const nlohmann::json &document, uint32_t seq_id) {
- Option<uint32_t> validation_op = validate_index_in_memory(document, seq_id);
-
- if(!validation_op.ok()) {
- return validation_op;
- }
-
- int32_t points = 0;
-
- if(!token_ranking_field.empty()) {
- if(document[token_ranking_field].is_number_float()) {
- // serialize float to an integer and reverse the inverted range
- float n = document[token_ranking_field];
- memcpy(&points, &n, sizeof(int32_t));
- points ^= ((points >> (std::numeric_limits<int32_t>::digits - 1)) | INT32_MIN);
- points = -1 * (INT32_MAX - points);
- } else {
- points = document[token_ranking_field];
- }
- }
-
+Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t seq_id, int32_t points) {
+ // assumes that validation has already been done
for(const std::pair<std::string, field> & field_pair: search_schema) {
const std::string & field_name = field_pair.first;
art_tree *t = search_index.at(field_name);
@@ -289,7 +108,7 @@ Option<uint32_t> Collection::index_in_memory(const nlohmann::json &document, uin
return Option<>(200);
}
-void Collection::index_int32_field(const int32_t value, uint32_t score, art_tree *t, uint32_t seq_id) const {
+void Index::index_int32_field(const int32_t value, uint32_t score, art_tree *t, uint32_t seq_id) const {
const int KEY_LEN = 8;
unsigned char key[KEY_LEN];
@@ -312,7 +131,7 @@ void Collection::index_int32_field(const int32_t value, uint32_t score, art_tree
art_insert(t, key, KEY_LEN, &art_doc, num_hits);
}
-void Collection::index_int64_field(const int64_t value, uint32_t score, art_tree *t, uint32_t seq_id) const {
+void Index::index_int64_field(const int64_t value, uint32_t score, art_tree *t, uint32_t seq_id) const {
const int KEY_LEN = 8;
unsigned char key[KEY_LEN];
@@ -335,7 +154,7 @@ void Collection::index_int64_field(const int64_t value, uint32_t score, art_tree
art_insert(t, key, KEY_LEN, &art_doc, num_hits);
}
-void Collection::index_float_field(const float value, uint32_t score, art_tree *t, uint32_t seq_id) const {
+void Index::index_float_field(const float value, uint32_t score, art_tree *t, uint32_t seq_id) const {
const int KEY_LEN = 8;
unsigned char key[KEY_LEN];
@@ -359,7 +178,7 @@ void Collection::index_float_field(const float value, uint32_t score, art_tree *
}
-void Collection::index_string_field(const std::string & text, const uint32_t score, art_tree *t,
+void Index::index_string_field(const std::string & text, const uint32_t score, art_tree *t,
uint32_t seq_id, const bool verbatim) const {
std::vector<std::string> tokens;
std::unordered_map<std::string, std::vector<uint32_t>> token_to_offsets;
@@ -405,35 +224,35 @@ void Collection::index_string_field(const std::string & text, const uint32_t sco
}
}
-void Collection::index_string_array_field(const std::vector<std::string> & strings, const uint32_t score, art_tree *t,
+void Index::index_string_array_field(const std::vector<std::string> & strings, const uint32_t score, art_tree *t,
uint32_t seq_id, const bool verbatim) const {
for(const std::string & str: strings) {
index_string_field(str, score, t, seq_id, verbatim);
}
}
-void Collection::index_int32_array_field(const std::vector<int32_t> & values, const uint32_t score, art_tree *t,
+void Index::index_int32_array_field(const std::vector<int32_t> & values, const uint32_t score, art_tree *t,
uint32_t seq_id) const {
for(const int32_t value: values) {
index_int32_field(value, score, t, seq_id);
}
}
-void Collection::index_int64_array_field(const std::vector<int64_t> & values, const uint32_t score, art_tree *t,
+void Index::index_int64_array_field(const std::vector<int64_t> & values, const uint32_t score, art_tree *t,
uint32_t seq_id) const {
for(const int64_t value: values) {
index_int64_field(value, score, t, seq_id);
}
}
-void Collection::index_float_array_field(const std::vector<float> & values, const uint32_t score, art_tree *t,
+void Index::index_float_array_field(const std::vector<float> & values, const uint32_t score, art_tree *t,
uint32_t seq_id) const {
for(const float value: values) {
index_float_field(value, score, t, seq_id);
}
}
-void Collection::do_facets(std::vector<facet> & facets, uint32_t* result_ids, size_t results_size) {
+void Index::do_facets(std::vector<facet> & facets, uint32_t* result_ids, size_t results_size) {
for(auto & a_facet: facets) {
// assumed that facet fields have already been validated upstream
const field & facet_field = facet_schema.at(a_facet.field_name);
@@ -453,7 +272,7 @@ void Collection::do_facets(std::vector<facet> & facets, uint32_t* result_ids, si
}
}
-void Collection::search_candidates(uint32_t* filter_ids, size_t filter_ids_length, std::vector<facet> & facets,
+void Index::search_candidates(uint32_t* filter_ids, size_t filter_ids_length, std::vector<facet> & facets,
const std::vector<sort_by> & sort_fields, int & candidate_rank,
std::vector<std::vector<art_leaf*>> & token_to_candidates,
std::vector<std::vector<art_leaf*>> & searched_queries, Topster<100> & topster,
@@ -499,7 +318,7 @@ void Collection::search_candidates(uint32_t* filter_ids, size_t filter_ids_lengt
result_size, &filtered_result_ids);
uint32_t* new_all_result_ids;
- all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, filtered_result_ids,
+ all_result_ids_len += ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, filtered_result_ids,
filtered_results_size, &new_all_result_ids);
delete [] *all_result_ids;
*all_result_ids = new_all_result_ids;
@@ -516,7 +335,7 @@ void Collection::search_candidates(uint32_t* filter_ids, size_t filter_ids_lengt
do_facets(facets, result_ids, result_size);
uint32_t* new_all_result_ids;
- all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, result_ids,
+ all_result_ids_len += ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, result_ids,
result_size, &new_all_result_ids);
delete [] *all_result_ids;
*all_result_ids = new_all_result_ids;
@@ -539,7 +358,7 @@ void Collection::search_candidates(uint32_t* filter_ids, size_t filter_ids_lengt
}
}
-size_t Collection::union_of_ids(std::vector<std::pair<uint32_t*, size_t>> & result_array_pairs,
+size_t Index::union_of_ids(std::vector<std::pair<uint32_t*, size_t>> & result_array_pairs,
uint32_t **results_out) {
uint32_t *results = nullptr;
size_t results_length = 0;
@@ -559,7 +378,7 @@ size_t Collection::union_of_ids(std::vector<std::pair<uint32_t*, size_t>> & resu
return results_length;
}
-Option<uint32_t> Collection::do_filtering(uint32_t** filter_ids_out, const std::string & simple_filter_str) {
+Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::string & simple_filter_str) {
// parse the filter string
std::vector<std::string> filter_blocks;
StringUtils::split(simple_filter_str, filter_blocks, "&&");
@@ -744,94 +563,28 @@ Option<uint32_t> Collection::do_filtering(uint32_t** filter_ids_out, const std::
return Option<>(filter_ids_length);
}
-Option<nlohmann::json> Collection::search(std::string query, const std::vector<std::string> search_fields,
- const std::string & simple_filter_query, const std::vector<std::string> & facet_fields,
- const std::vector<sort_by> & sort_fields, const int num_typos,
- const size_t per_page, const size_t page,
- const token_ordering token_order, const bool prefix) {
- nlohmann::json result = nlohmann::json::object();
- std::vector<facet> facets;
-
- // validate search fields
- for(const std::string & field_name: search_fields) {
- if(search_schema.count(field_name) == 0) {
- std::string error = "Could not find a field named `" + field_name + "` in the schema.";
- return Option<nlohmann::json>(400, error);
- }
-
- field search_field = search_schema.at(field_name);
- if(search_field.type != field_types::STRING && search_field.type != field_types::STRING_ARRAY) {
- std::string error = "Field `" + field_name + "` should be a string or a string array.";
- return Option<nlohmann::json>(400, error);
- }
-
- if(search_field.facet) {
- std::string error = "Field `" + field_name + "` is a faceted field - it cannot be used as a query field.";
- return Option<nlohmann::json>(400, error);
- }
- }
-
- // validate facet fields
- for(const std::string & field_name: facet_fields) {
- if(facet_schema.count(field_name) == 0) {
- std::string error = "Could not find a facet field named `" + field_name + "` in the schema.";
- return Option<nlohmann::json>(400, error);
- }
- facets.push_back(facet(field_name));
- }
-
- // validate sort fields and standardize
-
- std::vector<sort_by> sort_fields_std;
-
- for(const sort_by & _sort_field: sort_fields) {
- if(sort_index.count(_sort_field.name) == 0) {
- std::string error = "Could not find a field named `" + _sort_field.name + "` in the schema for sorting.";
- return Option<nlohmann::json>(400, error);
- }
-
- std::string sort_order = _sort_field.order;
- StringUtils::toupper(sort_order);
-
- if(sort_order != sort_field_const::asc && sort_order != sort_field_const::desc) {
- std::string error = "Order for field` " + _sort_field.name + "` should be either ASC or DESC.";
- return Option<nlohmann::json>(400, error);
- }
-
- sort_fields_std.push_back({_sort_field.name, sort_order});
- }
-
- // check for valid pagination
- if(page < 1) {
- std::string message = "Page must be an integer of value greater than 0.";
- return Option<nlohmann::json>(422, message);
- }
-
- if((page * per_page) > MAX_RESULTS) {
- std::string message = "Only the first " + std::to_string(MAX_RESULTS) + " results are available.";
- return Option<nlohmann::json>(422, message);
- }
+Option<size_t> Index::search(std::string query, const std::vector<std::string> search_fields,
+ const std::string & simple_filter_query, std::vector<facet> facets,
+ std::vector<sort_by> sort_fields_std, const int num_typos,
+ const size_t per_page, const size_t page, const token_ordering token_order,
+ const bool prefix, std::vector<std::pair<int, Topster<100>::KV>> & field_order_kvs,
+ size_t & all_result_ids_len, std::vector<std::vector<art_leaf*>> & searched_queries) {
const size_t num_results = (page * per_page);
- // process the filters
+ // process the filters first
+
uint32_t* filter_ids = nullptr;
Option<uint32_t> op_filter_ids_length = do_filtering(&filter_ids, simple_filter_query);
if(!op_filter_ids_length.ok()) {
- return Option<nlohmann::json>(op_filter_ids_length.code(), op_filter_ids_length.error());
+ return Option<size_t>(op_filter_ids_length.code(), op_filter_ids_length.error());
}
const uint32_t filter_ids_length = op_filter_ids_length.get();
// Order of `fields` are used to sort results
auto begin = std::chrono::high_resolution_clock::now();
- std::vector<std::pair<int, Topster<100>::KV>> field_order_kvs;
uint32_t* all_result_ids = nullptr;
- size_t all_result_ids_len = 0;
-
- // all search queries that were used for generating the results
- std::vector<std::vector<art_leaf*>> searched_queries;
- int searched_queries_index = 0;
for(int i = 0; i < search_fields.size(); i++) {
Topster<100> topster;
@@ -839,7 +592,7 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
// proceed to query search only when no filters are provided or when filtering produces results
if(simple_filter_query.size() == 0 || filter_ids_length > 0) {
search_field(query, field, filter_ids, filter_ids_length, facets, sort_fields_std, num_typos, num_results,
- searched_queries, searched_queries_index, topster, &all_result_ids, all_result_ids_len, token_order, prefix);
+ searched_queries, topster, &all_result_ids, all_result_ids_len, token_order, prefix);
topster.sort();
}
@@ -852,146 +605,10 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
delete [] filter_ids;
delete [] all_result_ids;
- // All fields are sorted descending
- std::sort(field_order_kvs.begin(), field_order_kvs.end(),
- [](const std::pair<int, Topster<100>::KV> & a, const std::pair<int, Topster<100>::KV> & b) {
- return std::tie(a.second.match_score, a.second.primary_attr, a.second.secondary_attr, a.first, a.second.key) >
- std::tie(b.second.match_score, b.second.primary_attr, b.second.secondary_attr, b.first, b.second.key);
- });
-
- result["hits"] = nlohmann::json::array();
- result["found"] = all_result_ids_len;
-
- const int start_result_index = (page - 1) * per_page;
- const int kvsize = field_order_kvs.size();
-
- if(start_result_index > (kvsize - 1)) {
- return Option<nlohmann::json>(result);
- }
-
- const int end_result_index = std::min(int(page * per_page), kvsize) - 1;
-
- for(size_t field_order_kv_index = start_result_index; field_order_kv_index <= end_result_index; field_order_kv_index++) {
- const auto & field_order_kv = field_order_kvs[field_order_kv_index];
- const std::string& seq_id_key = get_seq_id_key((uint32_t) field_order_kv.second.key);
-
- std::string value;
- store->get(seq_id_key, value);
-
- nlohmann::json document;
- try {
- document = nlohmann::json::parse(value);
- } catch(...) {
- return Option<nlohmann::json>(500, "Error while parsing stored document.");
- }
-
- // highlight query words in the result
- const std::string & field_name = search_fields[search_fields.size() - field_order_kv.first];
- field search_field = search_schema.at(field_name);
-
- // only string fields are supported for now
- if(search_field.type == field_types::STRING) {
- std::vector<std::string> tokens;
- StringUtils::split(document[field_name], tokens, " ");
-
- // positions in the document of each token in the query
- std::vector<std::vector<uint16_t>> token_positions;
-
- for (const art_leaf *token_leaf : searched_queries[field_order_kv.second.query_index]) {
- std::vector<uint16_t> positions;
- int doc_index = token_leaf->values->ids.indexOf(field_order_kv.second.key);
- if(doc_index == token_leaf->values->ids.getLength()) {
- continue;
- }
-
- uint32_t start_offset = token_leaf->values->offset_index.at(doc_index);
- uint32_t end_offset = (doc_index == token_leaf->values->ids.getLength() - 1) ?
- token_leaf->values->offsets.getLength() :
- token_leaf->values->offset_index.at(doc_index+1);
-
- while(start_offset < end_offset) {
- positions.push_back((uint16_t) token_leaf->values->offsets.at(start_offset));
- start_offset++;
- }
-
- token_positions.push_back(positions);
- }
-
- MatchScore mscore = MatchScore::match_score(field_order_kv.second.key, token_positions);
-
- // unpack `mscore.offset_diffs` into `token_indices`
- std::vector<size_t> token_indices;
- char num_tokens_found = mscore.offset_diffs[0];
- for(size_t i = 1; i <= num_tokens_found; i++) {
- if(mscore.offset_diffs[i] != std::numeric_limits<int8_t>::max()) {
- size_t token_index = (size_t)(mscore.start_offset + mscore.offset_diffs[i]);
- token_indices.push_back(token_index);
- }
- }
-
- auto minmax = std::minmax_element(token_indices.begin(), token_indices.end());
-
- // For longer strings, pick surrounding tokens within N tokens of min_index and max_index for the snippet
- const size_t start_index = (tokens.size() <= SNIPPET_STR_ABOVE_LEN) ? 0 :
- std::max(0, (int)(*(minmax.first)-5));
-
- const size_t end_index = (tokens.size() <= SNIPPET_STR_ABOVE_LEN) ? tokens.size() :
- std::min((int)tokens.size(), (int)(*(minmax.second)+5));
-
- for(const size_t token_index: token_indices) {
- tokens[token_index] = "<mark>" + tokens[token_index] + "</mark>";
- }
-
- std::stringstream snippet_stream;
- for(size_t snippet_index = start_index; snippet_index < end_index; snippet_index++) {
- if(snippet_index != start_index) {
- snippet_stream << " ";
- }
-
- snippet_stream << tokens[snippet_index];
- }
-
- document["_highlight"] = nlohmann::json::object();
- document["_highlight"][field_name] = snippet_stream.str();
- }
-
- result["hits"].push_back(document);
- }
-
- result["facet_counts"] = nlohmann::json::array();
-
- // populate facets
- for(const facet & a_facet: facets) {
- nlohmann::json facet_result = nlohmann::json::object();
- facet_result["field_name"] = a_facet.field_name;
- facet_result["counts"] = nlohmann::json::array();
-
- // keep only top 10 facets
- std::vector<std::pair<std::string, size_t>> value_to_count;
- for (auto itr = a_facet.result_map.begin(); itr != a_facet.result_map.end(); ++itr) {
- value_to_count.push_back(*itr);
- }
-
- std::sort(value_to_count.begin(), value_to_count.end(),
- [=](std::pair<std::string, size_t>& a, std::pair<std::string, size_t>& b) {
- return a.second > b.second;
- });
-
- for(auto i = 0; i < std::min((size_t)10, value_to_count.size()); i++) {
- auto & kv = value_to_count[i];
- nlohmann::json facet_value_count = nlohmann::json::object();
- facet_value_count["value"] = kv.first;
- facet_value_count["count"] = kv.second;
- facet_result["counts"].push_back(facet_value_count);
- }
-
- result["facet_counts"].push_back(facet_result);
- }
-
long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();
//!std::cout << "Time taken for result calc: " << timeMillis << "us" << std::endl;
- //!store->print_memory_usage();
- return result;
+
+ return Option<size_t>(field_order_kvs.size());
}
/*
@@ -1003,16 +620,16 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
4. Intersect the lists to find docs that match each phrase
5. Sort the docs based on some ranking criteria
*/
-void Collection::search_field(std::string & query, const std::string & field, uint32_t *filter_ids, size_t filter_ids_length,
+void Index::search_field(std::string & query, const std::string & field, uint32_t *filter_ids, size_t filter_ids_length,
std::vector<facet> & facets, const std::vector<sort_by> & sort_fields, const int num_typos,
const size_t num_results, std::vector<std::vector<art_leaf*>> & searched_queries,
- int & searched_queries_index, Topster<100> &topster, uint32_t** all_result_ids, size_t & all_result_ids_len,
+ Topster<100> &topster, uint32_t** all_result_ids, size_t & all_result_ids_len,
const token_ordering token_order, const bool prefix) {
std::vector<std::string> tokens;
StringUtils::split(query, tokens, " ");
const int max_cost = (num_typos < 0 || num_typos > 2) ? 2 : num_typos;
- const size_t max_results = std::min(num_results, (size_t) Collection::MAX_RESULTS);
+ const size_t max_results = std::min(num_results, (size_t) Index::MAX_RESULTS);
size_t total_results = topster.size;
@@ -1158,12 +775,12 @@ void Collection::search_field(std::string & query, const std::string & field, ui
}
return search_field(truncated_query, field, filter_ids, filter_ids_length, facets, sort_fields, num_typos,
- num_results, searched_queries, candidate_rank, topster, all_result_ids, all_result_ids_len,
+ num_results, searched_queries, topster, all_result_ids, all_result_ids_len,
token_order, prefix);
}
}
-void Collection::log_leaves(const int cost, const std::string &token, const std::vector<art_leaf *> &leaves) const {
+void Index::log_leaves(const int cost, const std::string &token, const std::vector<art_leaf *> &leaves) const {
printf("Token: %s, cost: %d, candidates: \n", token.c_str(), cost);
for(auto i=0; i < leaves.size(); i++) {
printf("%.*s, ", leaves[i]->key_len, leaves[i]->key);
@@ -1174,7 +791,7 @@ void Collection::log_leaves(const int cost, const std::string &token, const std:
}
}
-void Collection::score_results(const std::vector<sort_by> & sort_fields, const int & query_index, const int & candidate_rank,
+void Index::score_results(const std::vector<sort_by> & sort_fields, const int & query_index, const int & candidate_rank,
Topster<100> & topster, const std::vector<art_leaf *> &query_suggestion,
const uint32_t *result_ids, const size_t result_size) const {
@@ -1270,7 +887,7 @@ void Collection::score_results(const std::vector<sort_by> & sort_fields, const i
}
}
-void Collection::populate_token_positions(const std::vector<art_leaf *> &query_suggestion,
+void Index::populate_token_positions(const std::vector<art_leaf *> &query_suggestion,
spp::sparse_hash_map<const art_leaf *, uint32_t *> &leaf_to_indices,
size_t result_index, std::vector<std::vector<uint16_t>> &token_positions) const {
// for each token in the query, find the positions that it appears in this document
@@ -1295,7 +912,7 @@ void Collection::populate_token_positions(const std::vector<art_leaf *> &query_s
}
}
-inline std::vector<art_leaf *> Collection::next_suggestion(const std::vector<std::vector<art_leaf *>> &token_leaves,
+inline std::vector<art_leaf *> Index::next_suggestion(const std::vector<std::vector<art_leaf *>> &token_leaves,
long long int n) {
std::vector<art_leaf*> query_suggestion(token_leaves.size());
@@ -1314,7 +931,7 @@ inline std::vector<art_leaf *> Collection::next_suggestion(const std::vector<std
return query_suggestion;
}
-void Collection::remove_and_shift_offset_index(sorted_array &offset_index, const uint32_t *indices_sorted,
+void Index::remove_and_shift_offset_index(sorted_array &offset_index, const uint32_t *indices_sorted,
const uint32_t indices_length) {
uint32_t *curr_array = offset_index.uncompress();
uint32_t *new_array = new uint32_t[offset_index.getLength()];
@@ -1347,49 +964,7 @@ void Collection::remove_and_shift_offset_index(sorted_array &offset_index, const
delete[] new_array;
}
-Option<nlohmann::json> Collection::get(const std::string & id) {
- std::string seq_id_str;
- StoreStatus status = store->get(get_doc_id_key(id), seq_id_str);
-
- if(status == StoreStatus::NOT_FOUND) {
- return Option<nlohmann::json>(404, "Could not find a document with id: " + id);
- }
-
- uint32_t seq_id = (uint32_t) std::stol(seq_id_str);
-
- std::string parsed_document;
- store->get(get_seq_id_key(seq_id), parsed_document);
-
- nlohmann::json document;
- try {
- document = nlohmann::json::parse(parsed_document);
- } catch(...) {
- return Option<nlohmann::json>(500, "Error while parsing stored document.");
- }
-
- return Option<nlohmann::json>(document);
-}
-
-Option<std::string> Collection::remove(const std::string & id, const bool remove_from_store) {
- std::string seq_id_str;
- StoreStatus status = store->get(get_doc_id_key(id), seq_id_str);
-
- if(status == StoreStatus::NOT_FOUND) {
- return Option<std::string>(404, "Could not find a document with id: " + id);
- }
-
- uint32_t seq_id = (uint32_t) std::stol(seq_id_str);
-
- std::string parsed_document;
- store->get(get_seq_id_key(seq_id), parsed_document);
-
- nlohmann::json document;
- try {
- document = nlohmann::json::parse(parsed_document);
- } catch(...) {
- return Option<std::string>(500, "Error while parsing stored document.");
- }
-
+Option<uint32_t> Index::remove(const uint32_t seq_id, nlohmann::json & document) {
for(auto & name_field: search_schema) {
// Go through all the field names and find the keys+values so that they can be removed from in-memory index
std::vector<std::string> tokens;
@@ -1500,91 +1075,5 @@ Option<std::string> Collection::remove(const std::string & id, const bool remove
field_doc_value_map.second->erase(seq_id);
}
- if(remove_from_store) {
- store->remove(get_doc_id_key(id));
- store->remove(get_seq_id_key(seq_id));
- }
-
- num_documents -= 1;
-
- return Option<std::string>(id);
-}
-
-std::string Collection::get_next_seq_id_key(const std::string & collection_name) {
- return std::string(COLLECTION_NEXT_SEQ_PREFIX) + "_" + collection_name;
-}
-
-std::string Collection::get_seq_id_key(uint32_t seq_id) {
- // We can't simply do std::to_string() because we want to preserve the byte order.
- // & 0xFF masks all but the lowest eight bits.
- unsigned char bytes[4];
- bytes[0] = (unsigned char) ((seq_id >> 24) & 0xFF);
- bytes[1] = (unsigned char) ((seq_id >> 16) & 0xFF);
- bytes[2] = (unsigned char) ((seq_id >> 8) & 0xFF);
- bytes[3] = (unsigned char) ((seq_id & 0xFF));
-
- return get_seq_id_collection_prefix() + "_" + std::string(bytes, bytes+4);
-}
-
-uint32_t Collection::deserialize_seq_id_key(std::string serialized_seq_id) {
- uint32_t seq_id = ((serialized_seq_id[0] & 0xFF) << 24) | ((serialized_seq_id[1] & 0xFF) << 16) |
- ((serialized_seq_id[2] & 0xFF) << 8) | (serialized_seq_id[3] & 0xFF);
- return seq_id;
-}
-
-std::string Collection::get_doc_id_key(const std::string & doc_id) {
- return std::to_string(collection_id) + "_" + DOC_ID_PREFIX + "_" + doc_id;
-}
-
-std::string Collection::get_name() {
- return name;
-}
-
-size_t Collection::get_num_documents() {
- return num_documents;
-}
-
-uint32_t Collection::get_collection_id() {
- return collection_id;
-}
-
-uint32_t Collection::doc_id_to_seq_id(std::string doc_id) {
- std::string seq_id_str;
- store->get(get_doc_id_key(doc_id), seq_id_str);
- uint32_t seq_id = (uint32_t) std::stoi(seq_id_str);
- return seq_id;
-}
-
-std::vector<std::string> Collection::get_facet_fields() {
- std::vector<std::string> facet_fields_copy;
- for(auto it = facet_schema.begin(); it != facet_schema.end(); ++it) {
- facet_fields_copy.push_back(it->first);
- }
-
- return facet_fields_copy;
-}
-
-std::vector<field> Collection::get_sort_fields() {
- std::vector<field> sort_fields_copy;
- for(auto it = sort_schema.begin(); it != sort_schema.end(); ++it) {
- sort_fields_copy.push_back(it->second);
- }
-
- return sort_fields_copy;
-}
-
-spp::sparse_hash_map<std::string, field> Collection::get_schema() {
- return search_schema;
-};
-
-std::string Collection::get_meta_key(const std::string & collection_name) {
- return std::string(COLLECTION_META_PREFIX) + "_" + collection_name;
-}
-
-std::string Collection::get_seq_id_collection_prefix() {
- return std::to_string(collection_id) + "_" + std::string(SEQ_ID_PREFIX);
-}
-
-std::string Collection::get_token_ranking_field() {
- return token_ranking_field;
+ return Option<uint32_t>(seq_id);
}
\ No newline at end of file