Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Refactor facet search to use hash based mapping.

kishorencdfbe3888d82d
diff --git a/include/collection.h b/include/collection.h
index 82ff5295..c97c9b36 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -133,7 +133,7 @@ private:
std::unordered_map<std::string, field> search_schema;
- std::unordered_map<std::string, field> facet_schema;
+ std::map<std::string, field> facet_schema; // std::map guarantees order of fields
std::unordered_map<std::string, field> sort_schema;
@@ -154,6 +154,16 @@ private:
void populate_overrides(std::string query, std::map<uint32_t, size_t> & id_pos_map,
std::vector<uint32_t> & included_ids, std::vector<uint32_t> & excluded_ids);
+ static bool facet_count_compare(const std::pair<uint64_t, facet_count>& a,
+ const std::pair<uint64_t, facet_count>& b) {
+ return std::tie(a.second.count, a.first) > std::tie(b.second.count, a.first);
+ }
+
+ static bool facet_count_str_compare(const std::pair<std::string, size_t>& a,
+ const std::pair<std::string, size_t>& b) {
+ return a.second > b.second;
+ }
+
public:
Collection() = delete;
@@ -229,6 +239,8 @@ public:
static uint32_t get_seq_id_from_key(const std::string & key);
+ Option<bool> get_document_from_store(const std::string & seq_id_key, nlohmann::json & document);
+
Option<uint32_t> index_in_memory(const nlohmann::json & document, uint32_t seq_id);
void par_index_in_memory(std::vector<std::vector<index_record>> & iter_batch,
diff --git a/include/field.h b/include/field.h
index 2d145ea1..a011b0fe 100644
--- a/include/field.h
+++ b/include/field.h
@@ -120,39 +120,17 @@ struct sort_by {
}
};
-struct facet {
- const std::string field_name;
- std::map<std::string, size_t> result_map;
-
- facet(const std::string field_name): field_name(field_name) {
-
- }
+struct facet_count {
+ uint32_t count;
+ uint32_t doc_id; // used to fetch the actual document and the value from store
+ uint32_t array_pos;
};
-struct facet_value {
- // use string to int mapping for saving memory
- spp::sparse_hash_map<std::string, uint32_t> value_index;
- spp::sparse_hash_map<uint32_t, std::string> index_value;
-
- // doc_id => facet_ids mapping
- spp::sparse_hash_map<uint32_t, std::vector<uint32_t>> doc_values;
-
- uint32_t get_value_index(const std::string & value) {
- if(value_index.count(value) != 0) {
- return value_index[value];
- }
+struct facet {
+ const std::string field_name;
+ std::map<uint64_t, facet_count> result_map;
- uint32_t new_index = value_index.size();
- value_index.emplace(value, new_index);
- index_value.emplace(new_index, value);
- return new_index;
- }
+ facet(const std::string & field_name): field_name(field_name) {
- void index_values(uint32_t doc_seq_id, const std::vector<std::string> & values) {
- std::vector<uint32_t> value_vec(values.size());
- for(size_t i = 0; i < values.size(); i++) {
- value_vec[i] = get_value_index(values[i]);
- }
- doc_values.emplace(doc_seq_id, value_vec);
}
};
\ No newline at end of file
diff --git a/include/index.h b/include/index.h
index 6806c2c2..f77fdbc2 100644
--- a/include/index.h
+++ b/include/index.h
@@ -118,13 +118,13 @@ private:
std::unordered_map<std::string, field> search_schema;
- std::unordered_map<std::string, field> facet_schema;
+ std::map<std::string, field> facet_schema; // std::map guarantees order of fields
std::unordered_map<std::string, field> sort_schema;
spp::sparse_hash_map<std::string, art_tree*> search_index;
- spp::sparse_hash_map<std::string, facet_value> facet_index;
+ spp::sparse_hash_map<uint32_t, std::vector<std::vector<uint64_t>>> facet_index_v2;
spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, number_t>*> sort_index;
@@ -141,7 +141,7 @@ private:
void do_facets(std::vector<facet> & facets, const uint32_t* result_ids, size_t results_size);
- void drop_facets(std::vector<facet> & facets, const std::vector<uint32_t> ids);
+ void drop_facets(std::vector<facet> & facets, const std::vector<uint32_t> & ids);
void search_field(const uint8_t & field_id, std::string & query,
const std::string & field, uint32_t *filter_ids, size_t filter_ids_length,
@@ -193,8 +193,8 @@ private:
public:
Index() = delete;
- Index(const std::string name, std::unordered_map<std::string, field> search_schema,
- std::unordered_map<std::string, field> facet_schema, std::unordered_map<std::string, field> sort_schema);
+ Index(const std::string name, const std::unordered_map<std::string, field> & search_schema,
+ std::map<std::string, field> facet_schema, std::unordered_map<std::string, field> sort_schema);
~Index();
@@ -230,13 +230,13 @@ public:
static Option<uint32_t> validate_index_in_memory(const nlohmann::json &document, uint32_t seq_id,
const std::string & default_sorting_field,
const std::unordered_map<std::string, field> & search_schema,
- const std::unordered_map<std::string, field> & facet_schema);
+ const std::map<std::string, field> & facet_schema);
static batch_index_result batch_memory_index(Index *index,
std::vector<index_record> & iter_batch,
const std::string & default_sorting_field,
const std::unordered_map<std::string, field> & search_schema,
- const std::unordered_map<std::string, field> & facet_schema);
+ const std::map<std::string, field> & facet_schema);
// for limiting number of results on multiple candidates / query rewrites
enum {SEARCH_LIMIT_NUM = 100};
@@ -270,5 +270,9 @@ public:
bool terminate; // used for interrupting the thread during tear down
search_args search_params;
+
+ static void
+ populate_array_token_positions(std::vector<std::vector<std::vector<uint16_t>>> &array_token_positions,
+ const art_leaf *token_leaf, uint32_t doc_index);
};
diff --git a/include/match_score.h b/include/match_score.h
index 8ff09068..8d30d222 100644
--- a/include/match_score.h
+++ b/include/match_score.h
@@ -93,14 +93,14 @@ struct Match {
}
/*
- * Given *sorted offsets* of each target token in a *single* document, generates a score that indicates:
+ * Given *sorted offsets* of each target token in a *single* document (token_offsets), generates a score indicating:
* a) How many tokens are present within a match window
* b) The proximity between the tokens within the match window
*
* We use a priority queue to read the offset vectors in a sorted manner, slide a window of a given size, and
* compute the max_match and min_displacement of target tokens across the windows.
*/
- static Match match(uint32_t doc_id, const std::vector<std::vector<uint16_t>> &token_offsets) {
+ static Match match(uint32_t doc_id, const std::vector<std::vector<uint16_t>> & token_offsets) {
std::priority_queue<TokenOffset, std::vector<TokenOffset>, TokenOffset> heap;
const size_t tokens_size = std::min(token_offsets.size(), WINDOW_SIZE);
diff --git a/include/string_utils.h b/include/string_utils.h
index bb1cba6a..bf517868 100644
--- a/include/string_utils.h
+++ b/include/string_utils.h
@@ -156,8 +156,6 @@ struct StringUtils {
void unicode_normalize(std::string& str) const;
- void unicode_normalize2(std::string& str) const;
-
/* https://stackoverflow.com/a/34571089/131050 */
static std::string base64_encode(const std::string &in) {
std::string out;
diff --git a/include/wyhash_v5.h b/include/wyhash_v5.h
new file mode 100644
index 00000000..c0fa1ecf
--- /dev/null
+++ b/include/wyhash_v5.h
@@ -0,0 +1,153 @@
+// Author: Wang Yi <[email protected]>
+#ifndef wyhash_version_5
+#define wyhash_version_5
+#include <stdint.h>
+#include <string.h>
+#if defined(_MSC_VER) && defined(_M_X64)
+#include <intrin.h>
+#pragma intrinsic(_umul128)
+#endif
+#if defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__clang__)
+#define _likely_(x) __builtin_expect(x,1)
+#else
+#define _likely_(x) (x)
+#endif
+const uint64_t _wyp[6]={0xa0761d6478bd642full,0xe7037ed1a0b428dbull,0x8ebc6af09c88c6e3ull,0x589965cc75374cc3ull,0x1d8e4e27c47d124full,0x72b22b96e169b471ull};//default secret
+static inline uint64_t _wyrotr(uint64_t v, unsigned k){ return (v>>k)|(v<<(64-k)); }
+static inline uint64_t _wymum(uint64_t A, uint64_t B){
+#ifdef UNOFFICIAL_WYHASH_32BIT // fast on 32 bit system
+ uint64_t hh=(A>>32)*(B>>32), hl=(A>>32)*(unsigned)B, lh=(unsigned)A*(B>>32), ll=(uint64_t)(unsigned)A*(unsigned)B;
+ return _wyrotr(hl,32)^_wyrotr(lh,32)^hh^ll;
+#else
+#ifdef __SIZEOF_INT128__
+ __uint128_t r=A; r*=B; return (r>>64)^r;
+#elif defined(_MSC_VER) && defined(_M_X64)
+ A=_umul128(A, B, &B); return A^B;
+#else
+ uint64_t ha=A>>32, hb=B>>32, la=(uint32_t)A, lb=(uint32_t)B, hi, lo;
+ uint64_t rh=ha*hb, rm0=ha*lb, rm1=hb*la, rl=la*lb, t=rl+(rm0<<32), c=t<rl;
+ lo=t+(rm1<<32); c+=lo<t; hi=rh+(rm0>>32)+(rm1>>32)+c; return hi^lo;
+#endif
+#endif
+}
+static inline uint64_t _wymix(uint64_t A, uint64_t B){
+#ifdef UNOFFICIAL_WYHASH_FAST //lose entropy with probability 2^-66 per byte
+ return _wymum(A,B);
+#else
+ return A^B^_wymum(A,B);
+#endif
+}
+static inline uint64_t wyrand(uint64_t *seed){ *seed+=_wyp[0]; return _wymum(*seed^_wyp[1],*seed); }
+static inline double wy2u01(uint64_t r){ const double _wynorm=1.0/(1ull<<52); return (r>>11)*_wynorm; }
+static inline double wy2gau(uint64_t r){ const double _wynorm=1.0/(1ull<<20); return ((r&0x1fffff)+((r>>21)&0x1fffff)+((r>>42)&0x1fffff))*_wynorm-3.0; }
+#ifndef WYHASH_LITTLE_ENDIAN
+#if defined(_WIN32) || defined(__LITTLE_ENDIAN__) || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
+#define WYHASH_LITTLE_ENDIAN 1
+#elif defined(__BIG_ENDIAN__) || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
+#define WYHASH_LITTLE_ENDIAN 0
+#endif
+#endif
+#if(WYHASH_LITTLE_ENDIAN)
+static inline uint64_t _wyr8(const uint8_t *p){ uint64_t v; memcpy(&v, p, 8); return v; }
+static inline uint64_t _wyr4(const uint8_t *p){ unsigned v; memcpy(&v, p, 4); return v; }
+#else
+#if defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__clang__)
+static inline uint64_t _wyr8(const uint8_t *p){ uint64_t v; memcpy(&v, p, 8); return __builtin_bswap64(v); }
+static inline uint64_t _wyr4(const uint8_t *p){ unsigned v; memcpy(&v, p, 4); return __builtin_bswap32(v); }
+#elif defined(_MSC_VER)
+static inline uint64_t _wyr8(const uint8_t *p){ uint64_t v; memcpy(&v, p, 8); return _byteswap_uint64(v);}
+static inline uint64_t _wyr4(const uint8_t *p){ unsigned v; memcpy(&v, p, 4); return _byteswap_ulong(v); }
+#endif
+#endif
+static inline uint64_t _wyr3(const uint8_t *p, unsigned k){ return (((uint64_t)p[0])<<16)|(((uint64_t)p[k>>1])<<8)|p[k-1]; }
+static inline uint64_t FastestHash(const void *key, size_t len, uint64_t seed){
+ const uint8_t *p=(const uint8_t*)key;
+ return _likely_(len>=4)?(_wyr4(p)+_wyr4(p+len-4))*(_wyr4(p+(len>>1)-2)^seed):(_likely_(len)?_wyr3(p,len)*(_wyp[0]^seed):seed);
+}
+static inline uint64_t _wyhash(const void* key, uint64_t len, uint64_t seed, const uint64_t secret[6]){
+ const uint8_t *p=(const uint8_t*)key; uint64_t i=len; seed^=secret[4];
+ if(_likely_(i<=64)){
+ label:
+ if(_likely_(i>=8)){
+ if(_likely_(i<=16)) return _wymix(_wyr8(p)^secret[0],_wyr8(p+i-8)^seed);
+ else if(_likely_(i<=32)) return _wymix(_wyr8(p)^secret[0],_wyr8(p+8)^seed)^_wymix(_wyr8(p+i-16)^secret[1],_wyr8(p+i-8)^seed);
+ else return _wymix(_wyr8(p)^secret[0],_wyr8(p+8)^seed)^_wymix(_wyr8(p+16)^secret[1],_wyr8(p+24)^seed)
+ ^_wymix(_wyr8(p+i-32)^secret[2],_wyr8(p+i-24)^seed)^_wymix(_wyr8(p+i-16)^secret[3],_wyr8(p+i-8)^seed);
+ }
+ else {
+ if(_likely_(i>=4)) return _wymix(_wyr4(p)^secret[0],_wyr4(p+i-4)^seed);
+ else return _wymix((_likely_(i)?_wyr3(p,i):0)^secret[0],seed);
+ }
+ }
+ uint64_t see1=seed, see2=seed, see3=seed;
+ for(; i>64; i-=64,p+=64){
+ seed=_wymix(_wyr8(p)^secret[0],_wyr8(p+8)^seed); see1=_wymix(_wyr8(p+16)^secret[1],_wyr8(p+24)^see1);
+ see2=_wymix(_wyr8(p+32)^secret[2],_wyr8(p+40)^see2); see3=_wymix(_wyr8(p+48)^secret[3],_wyr8(p+56)^see3);
+ }
+ seed^=see1^see2^see3;
+ goto label;
+}
+static inline uint64_t wyhash(const void* key, uint64_t len, uint64_t seed, const uint64_t secret[6]){ return _wymum(_wyhash(key,len,seed,secret)^len,secret[5]); }
+static inline void make_secret(uint64_t seed, uint64_t secret[6]){
+ uint8_t c[]= {15,23,27,29,30,39,43,45,46,51,53,54,57,58,60,71,75,77,78,83,85,86,89,90,92,99,101,102,105,106,108,113,114,116,120,135,139,141,142,147,149,150,153,154,156,163,165,166,169,170,172,177,178,180,184,195,197,198,201,202,204,209,210,212,216,225,226,228,232,240};
+ for(size_t i=0; i<6; i++){
+ uint8_t ok;
+ do{
+ ok=1; secret[i]=0;
+ for(size_t j=0; j<64; j+=8) secret[i]|=((uint64_t)c[wyrand(&seed)%sizeof(c)])<<j;
+ for(size_t j=0; j<i; j++)
+#if defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__clang__)
+ if(__builtin_popcountll(secret[i]^secret[j])!=32) ok=0;
+#elif defined(_MSC_VER)
+ if(_mm_popcnt_u64(secret[i]^secret[j])!=32) ok=0;
+#endif
+ if(!ok) continue;
+ for(size_t j=2; j<0x100000000ull; j++) if(secret[i]%j==0){ ok=0; break; }
+ }while(!ok);
+ }
+}
+static inline uint64_t wyhash64(uint64_t A, uint64_t B){ return _wymum(_wymum(A^_wyp[0],B^_wyp[1]),_wyp[2]); }
+typedef struct wyhash_context {
+ uint64_t secret[5];
+ uint64_t seed, see1, see2, see3;
+ uint8_t buffer[64];
+ uint8_t left; // always in [0, 64]
+ int loop;
+ uint64_t total;
+} wyhash_context_t;
+static inline void wyhash_init(wyhash_context_t *const __restrict ctx, const uint64_t seed, const uint64_t secret[5]){
+ memcpy(ctx->secret, secret, sizeof(ctx->secret));
+ ctx->seed=seed^secret[4]; ctx->see1=ctx->seed; ctx->see2=ctx->seed; ctx->see3=ctx->seed;
+ ctx->left=0; ctx->total=0; ctx->loop=0;
+}
+static inline uint64_t _wyhash_loop(wyhash_context_t *const __restrict ctx, const uint8_t *p, const uint64_t len){
+ uint64_t i = len; ctx->loop|=(i>64);
+ for(; i>64; i-=64,p+=64){
+ ctx->seed=_wymix(_wyr8(p)^ctx->secret[0],_wyr8(p+8)^ctx->seed);
+ ctx->see1=_wymix(_wyr8(p+16)^ctx->secret[1],_wyr8(p+24)^ctx->see1);
+ ctx->see2=_wymix(_wyr8(p+32)^ctx->secret[2],_wyr8(p+40)^ctx->see2);
+ ctx->see3=_wymix(_wyr8(p+48)^ctx->secret[3],_wyr8(p+56)^ctx->see3);
+ }
+ return len - i;
+}
+static inline void wyhash_update(wyhash_context_t *const __restrict ctx, const void* const key, uint64_t len){
+ ctx->total += len; // overflow for total length is ok
+ const uint8_t* p = (const uint8_t*)key;
+ uint8_t slots = 64 - ctx->left; // assert left <= 64
+ slots = len <= slots ? len : slots;
+ memcpy(ctx->buffer + ctx->left, p, slots);
+ p += slots;
+ len -= slots;
+ ctx->left += slots;
+ ctx->left -= _wyhash_loop(ctx, ctx->buffer, ctx->left + (len > 0));
+ const uint64_t consumed = _wyhash_loop(ctx, p, len);
+ p += consumed;
+ len -= consumed; // assert len <= 64
+ ctx->left = ctx->left > len ? ctx->left : (uint8_t)len;
+ memcpy(ctx->buffer, p, len);
+}
+static inline uint64_t wyhash_final(wyhash_context_t *const __restrict ctx){
+ if(_likely_(ctx->loop)) ctx->seed ^= ctx->see1 ^ ctx->see2 ^ ctx->see3;
+ return _wymum(_wyhash(ctx->buffer, ctx->left, ctx->seed ^ ctx->secret[4], ctx->secret)^ctx->total,ctx->secret[4]);
+}
+#endif
diff --git a/src/collection.cpp b/src/collection.cpp
index 8c293772..af8664b6 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -44,7 +44,6 @@ Collection::Collection(const std::string name, const uint32_t collection_id, con
search_schema.emplace(field.name, field);
if(field.is_facet()) {
- facet_value fvalue;
facet_schema.emplace(field.name, field);
}
@@ -456,7 +455,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
std::string error = "Could not find a facet field named `" + field_name + "` in the schema.";
return Option<nlohmann::json>(404, error);
}
- facets.push_back(facet(field_name));
+ facets.emplace_back(field_name);
}
// validate sort fields and standardize
@@ -477,11 +476,11 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
return Option<nlohmann::json>(400, error);
}
- sort_fields_std.push_back({_sort_field.name, sort_order});
+ sort_fields_std.emplace_back(_sort_field.name, sort_order);
}
if(sort_fields_std.empty()) {
- sort_fields_std.push_back({default_sorting_field, sort_field_const::desc});
+ sort_fields_std.emplace_back(default_sorting_field, sort_field_const::desc);
}
// check for valid pagination
@@ -560,12 +559,14 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
if(acc_facet.result_map.count(facet_kv.first) == 0) {
// not found, so set it
- count = facet_kv.second;
+ count = facet_kv.second.count;
} else {
- count = acc_facet.result_map[facet_kv.first] + facet_kv.second;
+ count = acc_facet.result_map[facet_kv.first].count + facet_kv.second.count;
}
- acc_facet.result_map[facet_kv.first] = count;
+ acc_facet.result_map[facet_kv.first].count = count;
+ acc_facet.result_map[facet_kv.first].doc_id = facet_kv.second.doc_id;
+ acc_facet.result_map[facet_kv.first].array_pos = facet_kv.second.array_pos;
}
}
@@ -633,20 +634,12 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
const auto & field_order_kv = result_kvs[result_kvs_index];
const std::string& seq_id_key = get_seq_id_key((uint32_t) field_order_kv.key);
- std::string json_doc_str;
- StoreStatus json_doc_status = store->get(seq_id_key, json_doc_str);
-
- if(json_doc_status != StoreStatus::FOUND) {
- LOG(ERR) << "Could not locate the JSON document for sequence ID: " << seq_id_key;
- continue;
- }
-
nlohmann::json document;
+ const Option<bool> & document_op = get_document_from_store(seq_id_key, document);
- try {
- document = nlohmann::json::parse(json_doc_str);
- } catch(...) {
- return Option<nlohmann::json>(500, "Error while parsing stored document.");
+ if(!document_op.ok()) {
+ LOG(ERR) << "Document fetch error. " << document_op.error();
+ continue;
}
nlohmann::json wrapper_doc;
@@ -668,7 +661,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
std::sort(highlights.begin(), highlights.end());
- for(const auto highlight: highlights) {
+ for(const auto & highlight: highlights) {
nlohmann::json h_json = nlohmann::json::object();
h_json["field"] = highlight.field;
if(!highlight.indices.empty()) {
@@ -697,22 +690,49 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
facet_result["field_name"] = a_facet.field_name;
facet_result["counts"] = nlohmann::json::array();
- // keep only top 10 facets
- std::vector<std::pair<std::string, size_t>> value_to_count;
- for (auto itr = a_facet.result_map.begin(); itr != a_facet.result_map.end(); ++itr) {
- value_to_count.push_back(*itr);
+ std::vector<std::pair<uint64_t, facet_count>> facet_hash_counts;
+ for (const auto & itr : a_facet.result_map) {
+ facet_hash_counts.emplace_back(itr);
+ }
+
+ // keep only top K facets
+ auto max_facets = std::min(max_facet_values, facet_hash_counts.size());
+ std::nth_element(facet_hash_counts.begin(), facet_hash_counts.begin() + max_facets,
+ facet_hash_counts.end(), Collection::facet_count_compare);
+
+ std::vector<std::pair<std::string, size_t>> facet_counts;
+
+ for(size_t i = 0; i < max_facets; i++) {
+ // remap facet value hash with actual string
+ auto & kv = facet_hash_counts[i];
+
+ // fetch actual facet value from representative doc id
+ const std::string& seq_id_key = get_seq_id_key((uint32_t) kv.second.doc_id);
+ nlohmann::json document;
+ const Option<bool> & document_op = get_document_from_store(seq_id_key, document);
+
+ if(!document_op.ok()) {
+ LOG(ERR) << "Facet fetch error. " << document_op.error();
+ continue;
+ }
+
+ std::string facet_value;
+
+ if(facet_schema.at(a_facet.field_name).type == field_types::STRING) {
+ facet_value = document[a_facet.field_name];
+ } else if(facet_schema.at(a_facet.field_name).type == field_types::STRING_ARRAY) {
+ facet_value = document[a_facet.field_name][kv.second.array_pos];
+ }
+
+ facet_counts.emplace_back(std::make_pair(facet_value, kv.second.count));
}
- std::sort(value_to_count.begin(), value_to_count.end(),
- [=](std::pair<std::string, size_t>& a, std::pair<std::string, size_t>& b) {
- return a.second > b.second;
- });
+ std::stable_sort(facet_counts.begin(), facet_counts.end(), Collection::facet_count_str_compare);
- for(size_t i = 0; i < std::min(max_facet_values, value_to_count.size()); i++) {
- auto & kv = value_to_count[i];
+ for(const auto & facet_count: facet_counts) {
nlohmann::json facet_value_count = nlohmann::json::object();
- facet_value_count["value"] = kv.first;
- facet_value_count["count"] = kv.second;
+ facet_value_count["value"] = facet_count.first;
+ facet_value_count["count"] = facet_count.second;
facet_result["counts"].push_back(facet_value_count);
}
@@ -1055,4 +1075,21 @@ std::string Collection::get_seq_id_collection_prefix() {
std::string Collection::get_default_sorting_field() {
return default_sorting_field;
-}
\ No newline at end of file
+}
+
+Option<bool> Collection::get_document_from_store(const std::string &seq_id_key, nlohmann::json & document) {
+ std::string json_doc_str;
+ StoreStatus json_doc_status = store->get(seq_id_key, json_doc_str);
+
+ if(json_doc_status != StoreStatus::FOUND) {
+ return Option<bool>(500, "Could not locate the JSON document for sequence ID: " + seq_id_key);
+ }
+
+ try {
+ document = nlohmann::json::parse(json_doc_str);
+ } catch(...) {
+ return Option<bool>(500, "Error while parsing stored document with sequence ID: " + seq_id_key);
+ }
+
+ return Option<bool>(true);
+}
diff --git a/src/index.cpp b/src/index.cpp
index ce37a480..c33a4aa9 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -8,24 +8,21 @@
#include <match_score.h>
#include <string_utils.h>
#include <art.h>
+#include "wyhash_v5.h"
#include "logger.h"
-Index::Index(const std::string name, std::unordered_map<std::string, field> search_schema,
- std::unordered_map<std::string, field> facet_schema, std::unordered_map<std::string, field> sort_schema):
+Index::Index(const std::string name, const std::unordered_map<std::string, field> & search_schema,
+ std::map<std::string, field> facet_schema, std::unordered_map<std::string, field> sort_schema):
name(name), search_schema(search_schema), facet_schema(facet_schema), sort_schema(sort_schema) {
- for(const auto pair: search_schema) {
+ for(const auto & pair: search_schema) {
+ // NOTE: facet fields are also part of search schema
art_tree *t = new art_tree;
art_tree_init(t);
search_index.emplace(pair.first, t);
}
- for(const auto pair: facet_schema) {
- facet_value fvalue;
- facet_index.emplace(pair.first, fvalue);
- }
-
- for(const auto pair: sort_schema) {
+ for(const auto & pair: sort_schema) {
spp::sparse_hash_map<uint32_t, number_t> * doc_to_score = new spp::sparse_hash_map<uint32_t, number_t>();
sort_index.emplace(pair.first, doc_to_score);
}
@@ -83,7 +80,7 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t
if(field_pair.second.type == field_types::STRING) {
const std::string & text = document[field_name];
- index_string_field(text, points, t, seq_id, field_pair.second.is_facet());
+ index_string_field(text, points, t, seq_id, false);
} else if(field_pair.second.type == field_types::INT32) {
uint32_t value = document[field_name];
index_int32_field(value, points, t, seq_id);
@@ -98,7 +95,7 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t
index_bool_field(value, points, t, seq_id);
} else if(field_pair.second.type == field_types::STRING_ARRAY) {
std::vector<std::string> strings = document[field_name];
- index_string_array_field(strings, points, t, seq_id, field_pair.second.is_facet());
+ index_string_array_field(strings, points, t, seq_id, false);
} else if(field_pair.second.type == field_types::INT32_ARRAY) {
std::vector<int32_t> values = document[field_name];
index_int32_array_field(values, points, t, seq_id);
@@ -128,16 +125,27 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t
}
}
- for(const std::pair<std::string, field> & field_pair: facet_schema) {
+ std::vector<std::vector<uint64_t>> values(facet_schema.size());
+ facet_index_v2.emplace(seq_id, values);
+
+ size_t index_facet = 0;
+ for(const auto & field_pair: facet_schema) {
const std::string & field_name = field_pair.first;
- facet_value & fvalue = facet_index.at(field_name);
if(field_pair.second.type == field_types::STRING) {
- const std::string & value = document[field_name];
- fvalue.index_values(seq_id, { value });
+ const std::string & text = document[field_name].get<std::string>();
+ uint64_t hash = wyhash(text.c_str(), text.size(), 0, _wyp);
+ facet_index_v2[seq_id][index_facet].push_back(hash);
+ facet_index_v2[seq_id][index_facet].resize(facet_index_v2[seq_id][index_facet].size());
} else if(field_pair.second.type == field_types::STRING_ARRAY) {
- const std::vector<std::string> & values = document[field_name];
- fvalue.index_values(seq_id, values);
+ const std::vector<std::string> & texts = document[field_name];
+ for(auto & text: texts) {
+ uint64_t hash = wyhash(text.c_str(), text.size(), 0, _wyp);
+ facet_index_v2[seq_id][index_facet].push_back(hash);
+ }
+ facet_index_v2[seq_id][index_facet].resize(facet_index_v2[seq_id][index_facet].size());
}
+
+ index_facet++;
}
num_documents += 1;
@@ -147,7 +155,7 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t
Option<uint32_t> Index::validate_index_in_memory(const nlohmann::json &document, uint32_t seq_id,
const std::string & default_sorting_field,
const std::unordered_map<std::string, field> & search_schema,
- const std::unordered_map<std::string, field> & facet_schema) {
+ const std::map<std::string, field> & facet_schema) {
if(document.count(default_sorting_field) == 0) {
return Option<>(400, "Field `" + default_sorting_field + "` has been declared as a default sorting field, "
"but is not found in the document.");
@@ -272,7 +280,7 @@ Option<uint32_t> Index::validate_index_in_memory(const nlohmann::json &document,
batch_index_result Index::batch_memory_index(Index *index, std::vector<index_record> & iter_batch,
const std::string & default_sorting_field,
const std::unordered_map<std::string, field> & search_schema,
- const std::unordered_map<std::string, field> & facet_schema) {
+ const std::map<std::string, field> & facet_schema) {
batch_index_result result;
@@ -515,40 +523,50 @@ void Index::index_float_array_field(const std::vector<float> & values, const uin
}
void Index::do_facets(std::vector<facet> & facets, const uint32_t* result_ids, size_t results_size) {
+ std::map<std::string, size_t> facet_to_index;
+
+ size_t i_facet = 0;
+ for(const auto & facet: facet_schema) {
+ facet_to_index[facet.first] = i_facet;
+ i_facet++;
+ }
+
for(auto & a_facet: facets) {
// assumed that facet fields have already been validated upstream
- const field & facet_field = facet_schema.at(a_facet.field_name);
- const facet_value & fvalue = facet_index.at(facet_field.name);
-
for(size_t i = 0; i < results_size; i++) {
uint32_t doc_seq_id = result_ids[i];
- if(fvalue.doc_values.count(doc_seq_id) != 0) {
- // for every result document, get the facet values and increment counter
- const std::vector<uint32_t> & value_indices = fvalue.doc_values.at(doc_seq_id);
- for(size_t j = 0; j < value_indices.size(); j++) {
- const std::string & facet_value = fvalue.index_value.at(value_indices.at(j));
- a_facet.result_map[facet_value] += 1;
+ if(facet_index_v2.count(doc_seq_id) != 0) {
+ const std::vector<uint64_t> & values = facet_index_v2[doc_seq_id][facet_to_index[a_facet.field_name]];
+
+ for(size_t j = 0; j < values.size(); j++) {
+ a_facet.result_map[values[j]].count += 1;
+ a_facet.result_map[values[j]].doc_id = doc_seq_id;
+ a_facet.result_map[values[j]].array_pos = j;
}
}
}
}
}
-void Index::drop_facets(std::vector<facet> & facets, const std::vector<uint32_t> ids) {
+void Index::drop_facets(std::vector<facet> & facets, const std::vector<uint32_t> & ids) {
+ std::map<std::string, size_t> facet_to_index;
+
+ size_t i_facet = 0;
+ for(const auto & facet: facet_schema) {
+ facet_to_index[facet.first] = i_facet;
+ i_facet++;
+ }
+
for(auto & a_facet: facets) {
// assumed that facet fields have already been validated upstream
- const field & facet_field = facet_schema.at(a_facet.field_name);
- const facet_value & fvalue = facet_index.at(facet_field.name);
-
for(const uint32_t doc_seq_id: ids) {
- if(fvalue.doc_values.count(doc_seq_id) != 0) {
- // for every result document, get the values associated and decrement counter
- const std::vector<uint32_t> & value_indices = fvalue.doc_values.at(doc_seq_id);
- for(size_t j = 0; j < value_indices.size(); j++) {
- const std::string & facet_value = fvalue.index_value.at(value_indices.at(j));
- a_facet.result_map[facet_value] -= 1;
- if(a_facet.result_map[facet_value] == 0) {
- a_facet.result_map.erase(facet_value);
+ if(facet_index_v2.count(doc_seq_id) != 0) {
+ const std::vector<uint64_t> & values = facet_index_v2[doc_seq_id][facet_to_index[a_facet.field_name]];
+
+ for(size_t j = 0; j < values.size(); j++) {
+ a_facet.result_map[values[j]].count -= 1;
+ if(a_facet.result_map[values[j]].count == 0) {
+ a_facet.result_map.erase(values[j]);
}
}
}
@@ -715,21 +733,14 @@ Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::vecto
} else if(f.is_string()) {
for(const std::string & filter_value: a_filter.values) {
std::vector<std::string> str_tokens;
-
- bool is_facet_field = (facet_index.count(a_filter.field_name) != 0);
- std::string delim = is_facet_field ? "" : " ";
- StringUtils::split(filter_value, str_tokens, delim);
+ StringUtils::split(filter_value, str_tokens, " ");
uint32_t* filtered_ids = nullptr;
size_t filtered_size = 0;
for(size_t i = 0; i < str_tokens.size(); i++) {
std::string & str_token = str_tokens[i];
-
- if(!is_facet_field) {
- // We should not normalize when the filter field is also a facet field
- string_utils.unicode_normalize(str_token);
- }
+ string_utils.unicode_normalize(str_token);
art_leaf* leaf = (art_leaf *) art_search(t, (const unsigned char*) str_token.c_str(),
str_token.length()+1);
@@ -752,7 +763,7 @@ Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::vecto
}
}
- filter_result_array_pairs.push_back(std::make_pair(filtered_ids, filtered_size));
+ filter_result_array_pairs.emplace_back(filtered_ids, filtered_size);
}
}
@@ -1332,38 +1343,43 @@ void Index::populate_token_positions(const std::vector<art_leaf *> &query_sugges
continue;
}
- uint32_t start_offset = token_leaf->values->offset_index.at(doc_index);
- uint32_t end_offset = (doc_index == token_leaf->values->ids.getLength() - 1) ?
- token_leaf->values->offsets.getLength() :
- token_leaf->values->offset_index.at(doc_index+1);
-
- std::vector<uint16_t> positions;
- uint16_t prev_pos = -1;
-
- while(start_offset < end_offset) {
- auto pos = (uint16_t) token_leaf->values->offsets.at(start_offset);
- start_offset++;
-
- if(pos == prev_pos) { // indicates end of array index
- if(!positions.empty()) {
- size_t array_index = (uint16_t) token_leaf->values->offsets.at(start_offset);
- array_token_positions[array_index].push_back(positions);
- positions.clear();
- }
+ populate_array_token_positions(array_token_positions, token_leaf, doc_index);
+ }
+}
- start_offset++; // skip current value which is array index
- prev_pos = -1;
- continue;
+void Index::populate_array_token_positions(std::vector<std::vector<std::vector<uint16_t>>> & array_token_positions,
+ const art_leaf *token_leaf, uint32_t doc_index) {
+ uint32_t start_offset = token_leaf->values->offset_index.at(doc_index);
+ uint32_t end_offset = (doc_index == token_leaf->values->ids.getLength() - 1) ?
+ token_leaf->values->offsets.getLength() :
+ token_leaf->values->offset_index.at(doc_index+1);
+
+ std::vector<uint16_t> positions;
+ uint16_t prev_pos = -1;
+
+ while(start_offset < end_offset) {
+ auto pos = (uint16_t) token_leaf->values->offsets.at(start_offset);
+ start_offset++;
+
+ if(pos == prev_pos) { // indicates end of array index
+ if(!positions.empty()) {
+ size_t array_index = (uint16_t) token_leaf->values->offsets.at(start_offset);
+ array_token_positions[array_index].push_back(positions);
+ positions.clear();
}
- prev_pos = pos;
- positions.push_back(pos);
+ start_offset++; // skip current value which is array index
+ prev_pos = -1;
+ continue;
}
- if(!positions.empty()) {
- // for plain string fields
- array_token_positions[0].push_back(positions);
- }
+ prev_pos = pos;
+ positions.push_back(pos);
+ }
+
+ if(!positions.empty()) {
+ // for plain string fields
+ array_token_positions[0].push_back(positions);
}
}
@@ -1536,9 +1552,7 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, nlohmann::json & document)
}
// remove facets if any
- for(auto & field_facet_value: facet_index) {
- field_facet_value.second.doc_values.erase(seq_id);
- }
+ facet_index_v2.erase(seq_id);
// remove sort index if any
for(auto & field_doc_value_map: sort_index) {
diff --git a/src/string_utils.cpp b/src/string_utils.cpp
index d9b4c916..53a35a6c 100644
--- a/src/string_utils.cpp
+++ b/src/string_utils.cpp
@@ -48,24 +48,3 @@ void StringUtils::unicode_normalize(std::string & str) const {
str.assign(lower_and_no_special_chars(out.str()));
}
-
-void StringUtils::unicode_normalize2(std::string& str) const {
- size_t outbuflen = str.length() * 2;
- char output[outbuflen];
- char *outptr = output;
-
- char *input = (char *) str.c_str();
- size_t insize = str.length();
- size_t outsize = outbuflen;
-
- iconv(cd, &input, &insize, &outptr, &outsize);
- size_t actual_size = outbuflen - outsize;
-
- if(actual_size == 0) {
- str.assign(lower_and_no_special_chars(str));
- return ;
- }
-
- std::string nstr = std::string(output, actual_size);
- str.assign(lower_and_no_special_chars(nstr));
-}
\ No newline at end of file
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 3a8fe0ae..8391185a 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -1608,15 +1608,15 @@ TEST_F(CollectionTest, FilterOnTextFields) {
results = coll_array_fields->search("Jeremy", query_fields, "tags: bronze", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
ASSERT_EQ(2, results["hits"].size());
- // when comparators are used, should just treat them as part of search string
+ // when comparators are used, they should be ignored
results = coll_array_fields->search("Jeremy", query_fields, "tags:<bronze", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
- ASSERT_EQ(0, results["hits"].size());
+ ASSERT_EQ(2, results["hits"].size());
results = coll_array_fields->search("Jeremy", query_fields, "tags:<=BRONZE", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
- ASSERT_EQ(0, results["hits"].size());
+ ASSERT_EQ(2, results["hits"].size());
results = coll_array_fields->search("Jeremy", query_fields, "tags:>BRONZE", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
- ASSERT_EQ(0, results["hits"].size());
+ ASSERT_EQ(2, results["hits"].size());
collectionManager.drop_collection("coll_array_fields");
}
@@ -1717,16 +1717,16 @@ TEST_F(CollectionTest, FacetCounts) {
ASSERT_EQ("tags", results["facet_counts"][0]["field_name"]);
ASSERT_EQ(4, results["facet_counts"][0]["counts"].size());
- ASSERT_EQ("gold", results["facet_counts"][0]["counts"][0]["value"]);
+ ASSERT_STREQ("gold", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
ASSERT_EQ(3, (int) results["facet_counts"][0]["counts"][0]["count"]);
- ASSERT_EQ("silver", results["facet_counts"][0]["counts"][1]["value"]);
+ ASSERT_STREQ("silver", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
ASSERT_EQ(3, (int) results["facet_counts"][0]["counts"][1]["count"]);
- ASSERT_EQ("bronze", results["facet_counts"][0]["counts"][2]["value"]);
+ ASSERT_STREQ("bronze", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
ASSERT_EQ(2, (int) results["facet_counts"][0]["counts"][2]["count"]);
- ASSERT_EQ("FINE PLATINUM", results["facet_counts"][0]["counts"][3]["value"]);
+ ASSERT_STREQ("FINE PLATINUM", results["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());
ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][3]["count"]);
// facet with facet count limit
@@ -1776,10 +1776,10 @@ TEST_F(CollectionTest, FacetCounts) {
ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][2]["count"]);
ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][3]["count"]);
- ASSERT_EQ("silver", results["facet_counts"][0]["counts"][0]["value"]);
- ASSERT_EQ("FINE PLATINUM", results["facet_counts"][0]["counts"][1]["value"]);
- ASSERT_EQ("bronze", results["facet_counts"][0]["counts"][2]["value"]);
- ASSERT_EQ("gold", results["facet_counts"][0]["counts"][3]["value"]);
+ ASSERT_STREQ("silver", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("gold", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("FINE PLATINUM", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("bronze", results["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());
// facet with wildcard query
facets.clear();
@@ -1789,16 +1789,16 @@ TEST_F(CollectionTest, FacetCounts) {
ASSERT_EQ(3, results["hits"].size());
ASSERT_EQ(1, results["facet_counts"].size());
- ASSERT_EQ("tags", results["facet_counts"][0]["field_name"]);
+ ASSERT_STREQ("tags", results["facet_counts"][0]["field_name"].get<std::string>().c_str());
ASSERT_EQ(2, (int) results["facet_counts"][0]["counts"][0]["count"]);
ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][1]["count"]);
ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][2]["count"]);
ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][3]["count"]);
- ASSERT_EQ("silver", results["facet_counts"][0]["counts"][0]["value"]);
- ASSERT_EQ("FINE PLATINUM", results["facet_counts"][0]["counts"][1]["value"]);
- ASSERT_EQ("bronze", results["facet_counts"][0]["counts"][2]["value"]);
- ASSERT_EQ("gold", results["facet_counts"][0]["counts"][3]["value"]);
+ ASSERT_STREQ("silver", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("gold", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("FINE PLATINUM", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("bronze", results["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());
collectionManager.drop_collection("coll_array_fields");
}