Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Further improvements to filter performance

Jason Bosco3d8f6fe43f3d
diff --git a/include/sorted_array.h b/include/sorted_array.h
index 79e3a3ce..9e9ada90 100644
--- a/include/sorted_array.h
+++ b/include/sorted_array.h
@@ -31,14 +31,23 @@ private:
uint32_t lower_bound_search_bits(const uint8_t *in, uint32_t imin, uint32_t imax, uint32_t base,
uint32_t bits, uint32_t value, uint32_t *actual);
+ uint32_t lower_bound_search(const uint32_t *in, uint32_t imin, uint32_t imax,
+ uint32_t value, uint32_t *actual);
+
void binary_search_indices(const uint32_t *values, int low_vindex, int high_vindex,
int low_index, int high_index, uint32_t base, uint32_t bits,
uint32_t *indices);
+ void binary_search_indices(const uint32_t *values, int low_vindex, int high_vindex,
+ int low_index, int high_index, uint32_t *indices);
+
void binary_count_indices(const uint32_t *values, int low_vindex, int high_vindex,
int low_index, int high_index, uint32_t base, uint32_t bits,
size_t& num_found);
+ void binary_count_indices(const uint32_t *values, int low_vindex, int high_vindex,
+ const uint32_t* src, int low_index, int high_index, size_t& num_found);
+
public:
void load(const uint32_t *sorted_array, const uint32_t array_length);
diff --git a/src/art.cpp b/src/art.cpp
index 0703dfe8..836100c4 100644
--- a/src/art.cpp
+++ b/src/art.cpp
@@ -1421,7 +1421,7 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,
//long long int time_micro = microseconds(std::chrono::high_resolution_clock::now() - begin).count();
//!LOG(INFO) << "Time taken for fuzz: " << time_micro << "us, size of nodes: " << nodes.size();
- //begin = std::chrono::high_resolution_clock::now();
+ //auto begin = std::chrono::high_resolution_clock::now();
for(auto node: nodes) {
art_topk_iter(node, token_order, max_words, filter_ids, filter_ids_length, results);
@@ -1433,8 +1433,14 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,
std::sort(results.begin(), results.end(), compare_art_leaf_score);
}
- //time_micro = microseconds(std::chrono::high_resolution_clock::now() - begin).count();
- //!LOG(INFO) << "Time taken for art_topk_iter: " << time_micro << "us";
+ /*auto time_micro = microseconds(std::chrono::high_resolution_clock::now() - begin).count();
+
+ if(time_micro > 1000) {
+ LOG(INFO) << "Time taken for art_topk_iter: " << time_micro
+ << "us, size of nodes: " << nodes.size()
+ << ", filter_ids_length: " << filter_ids_length;
+ }*/
+
return 0;
}
diff --git a/src/index.cpp b/src/index.cpp
index 5e1b7621..a21352bb 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -952,6 +952,8 @@ void Index::search_candidates(const uint8_t & field_id,
}
Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::vector<filter> & filters) {
+ //auto begin = std::chrono::high_resolution_clock::now();
+
uint32_t* filter_ids = nullptr;
uint32_t filter_ids_length = 0;
@@ -1137,6 +1139,11 @@ Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::vecto
}
}
+ /*long long int timeMillis =
+ std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();
+
+ LOG(INFO) << "Time taken for filtering: " << timeMillis << "ms";*/
+
*filter_ids_out = filter_ids;
return Option<>(filter_ids_length);
}
@@ -1319,6 +1326,8 @@ void Index::search(Option<uint32_t> & outcome,
std::vector<std::vector<KV*>> & override_result_kvs,
const size_t typo_tokens_threshold) {
+ //auto begin = std::chrono::high_resolution_clock::now();
+
// process the filters
uint32_t* filter_ids = nullptr;
@@ -1448,6 +1457,8 @@ void Index::search(Option<uint32_t> & outcome,
q_pos_synonyms.emplace_back(q_pos_syn);
}
+ //begin = std::chrono::high_resolution_clock::now();
+
// non-wildcard
for(size_t i = 0; i < num_search_fields; i++) {
// proceed to query search only when no filters are provided or when filtering produces results
@@ -1603,8 +1614,8 @@ void Index::search(Option<uint32_t> & outcome,
delete ftopster;
}
- //long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();
- //!LOG(INFO) << "Time taken for result calc: " << timeMillis << "us";
+ //long long int timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();
+ //LOG(INFO) << "Time taken for result calc: " << timeMillis << "ms";
outcome = Option<uint32_t>(1);
}
diff --git a/src/sorted_array.cpp b/src/sorted_array.cpp
index da6b3f25..43f37116 100644
--- a/src/sorted_array.cpp
+++ b/src/sorted_array.cpp
@@ -138,6 +138,37 @@ uint32_t sorted_array::lower_bound_search_bits(const uint8_t *in, uint32_t imin,
return imax;
}
+
+// returns the first element in the sequence which does not compare less than |value|.
+uint32_t sorted_array::lower_bound_search(const uint32_t *in, uint32_t imin, uint32_t imax,
+ uint32_t value, uint32_t *actual) {
+ uint32_t imid;
+ uint32_t v;
+
+ while (imin + 1 < imax) {
+ imid = imin + ((imax - imin) / 2);
+
+ v = in[imid];
+ if (v >= value) {
+ imax = imid;
+ }
+ else if (v < value) {
+ imin = imid;
+ }
+ }
+
+ v = in[imin];
+ if (v >= value) {
+ *actual = v;
+ return imin;
+ }
+
+ v = in[imax];
+ *actual = v;
+ return imax;
+}
+
+
void sorted_array::binary_search_indices(const uint32_t *values, int low_vindex, int high_vindex,
int low_index, int high_index, uint32_t base, uint32_t bits,
uint32_t *indices) {
@@ -159,6 +190,27 @@ void sorted_array::binary_search_indices(const uint32_t *values, int low_vindex,
}
}
+void sorted_array::binary_search_indices(const uint32_t *values, int low_vindex, int high_vindex, int low_index,
+ int high_index, uint32_t *indices) {
+
+ uint32_t actual_value = 0;
+
+ if(high_vindex >= low_vindex && high_index >= low_index) {
+ size_t pivot_vindex = (low_vindex + high_vindex) / 2;
+
+ uint32_t in_index = lower_bound_search(values, low_index, high_index,
+ values[pivot_vindex], &actual_value);
+ if(actual_value == values[pivot_vindex]) {
+ indices[pivot_vindex] = in_index;
+ } else {
+ indices[pivot_vindex] = length;
+ }
+
+ binary_search_indices(values, low_vindex, pivot_vindex-1, low_index, in_index, indices);
+ binary_search_indices(values, pivot_vindex+1, high_vindex, in_index, high_index, indices);
+ }
+}
+
void sorted_array::indexOf(const uint32_t *values, const size_t values_len, uint32_t *indices) {
if(values_len == 0) {
return ;
@@ -247,27 +299,49 @@ size_t sorted_array::numFoundOf(const uint32_t *values, const size_t values_len)
return num_found;
}
- uint32_t base = *(uint32_t *)(in + 0);
- uint32_t bits = *(in + 4);
-
uint32_t low_index, high_index;
uint32_t actual_value = 0;
- // identify the upper and lower bounds of the search space
- int head = -1;
- do {
- head++;
- low_index = lower_bound_search_bits(in+METADATA_OVERHEAD, 0, length-1, base, bits, values[head], &actual_value);
- } while(head < int(values_len - 1) && actual_value > values[head]);
+ if(length > values_len) {
+ uint32_t base = *(uint32_t *)(in + 0);
+ uint32_t bits = *(in + 4);
+
+ // identify the upper and lower bounds of the search space
+ int head = -1;
+ do {
+ head++;
+ low_index = lower_bound_search_bits(in+METADATA_OVERHEAD, 0, length-1, base, bits, values[head], &actual_value);
+ } while(head < int(values_len - 1) && actual_value > values[head]);
+
+ int tail = values_len;
+ do {
+ tail--;
+ high_index = lower_bound_search_bits(in+METADATA_OVERHEAD, 0, length-1, base, bits, values[tail], &actual_value);
+ } while(tail > 0 && actual_value < values[tail]);
+
+ // recursively search within the bounds for all values
+ binary_count_indices(values, head, tail, low_index, high_index, base, bits, num_found);
+ } else {
+ // identify the upper and lower bounds of the search space
+ uint32_t* src = uncompress(length);
- int tail = values_len;
- do {
- tail--;
- high_index = lower_bound_search_bits(in+METADATA_OVERHEAD, 0, length-1, base, bits, values[tail], &actual_value);
- } while(tail > 0 && actual_value < values[tail]);
+ int head = -1;
+ do {
+ head++;
+ low_index = lower_bound_search(values, 0, values_len-1, src[head], &actual_value);
+ } while(head < int(length - 1) && actual_value > src[head]);
- // recursively search within the bounds for all values
- binary_count_indices(values, head, tail, low_index, high_index, base, bits, num_found);
+ int tail = length;
+ do {
+ tail--;
+ high_index = lower_bound_search(values, 0, values_len-1, src[tail], &actual_value);
+ } while(tail > 0 && actual_value < src[tail]);
+
+ // recursively search within the bounds for all values
+ binary_count_indices(src, head, tail, values, low_index, high_index, num_found);
+
+ delete [] src;
+ }
return num_found;
}
@@ -278,7 +352,7 @@ void sorted_array::binary_count_indices(const uint32_t *values, int low_vindex,
uint32_t actual_value = 0;
if(high_vindex >= low_vindex && high_index >= low_index) {
- size_t pivot_vindex = (low_vindex + high_vindex) / 2;
+ int pivot_vindex = (low_vindex + high_vindex) / 2;
uint32_t in_index = lower_bound_search_bits(in+METADATA_OVERHEAD, low_index, high_index, base, bits,
values[pivot_vindex], &actual_value);
@@ -293,3 +367,24 @@ void sorted_array::binary_count_indices(const uint32_t *values, int low_vindex,
binary_count_indices(values, pivot_vindex+1, high_vindex, in_index, high_index, base, bits, num_found);
}
}
+
+void sorted_array::binary_count_indices(const uint32_t *values, int low_vindex, int high_vindex,
+ const uint32_t* src, int low_index, int high_index, size_t &num_found) {
+
+ uint32_t actual_value = 0;
+
+ if(high_vindex >= low_vindex && high_index >= low_index) {
+ int pivot_vindex = (low_vindex + high_vindex) / 2;
+
+ uint32_t in_index = lower_bound_search(src, low_index, high_index, values[pivot_vindex], &actual_value);
+
+ //LOG(INFO) << "pivot_vindex: " << pivot_vindex << ", values[pivot_vindex]: " << values[pivot_vindex];
+ if(actual_value == values[pivot_vindex]) {
+ //LOG(INFO) << actual_value;
+ num_found++;
+ }
+
+ binary_count_indices(values, low_vindex, pivot_vindex-1, src, low_index, in_index, num_found);
+ binary_count_indices(values, pivot_vindex+1, high_vindex, src, in_index, high_index, num_found);
+ }
+}
diff --git a/test/sorted_array_test.cpp b/test/sorted_array_test.cpp
index 717da204..425c8336 100644
--- a/test/sorted_array_test.cpp
+++ b/test/sorted_array_test.cpp
@@ -341,7 +341,7 @@ TEST(SortedArrayTest, BulkIndexOf2) {
ASSERT_EQ(4, indices[6]);
}
-TEST(SortedArrayTest, NumFoundOf) {
+TEST(SortedArrayTest, NumFoundOfSortedArrayGreaterThanValues) {
std::vector<uint32_t> ids = {3, 44, 51, 54, 57, 60, 121, 136, 232, 238, 278, 447, 452, 454, 455, 456, 457, 459, 463,
465, 471, 472, 473, 474, 475, 478, 479, 480, 486, 490, 492, 496, 503, 508, 510, 512,
515, 526, 527, 533, 534, 537, 544, 547, 551, 553, 565, 573, 574, 577, 579, 617, 621,
@@ -389,6 +389,66 @@ TEST(SortedArrayTest, NumFoundOf) {
auto num_found = arr.numFoundOf(&filter_ids[0], filter_ids.size());
ASSERT_EQ(3, num_found);
+ filter_ids = {4,5,6,7,8};
+ ids = {1,2,3,4,6,7,8,9,10};
+
+ sorted_array arr2;
+ for(auto id: ids) {
+ arr2.append(id);
+ }
+
+ num_found = arr2.numFoundOf(&filter_ids[0], filter_ids.size());
+ ASSERT_EQ(4, num_found);
+}
+
+TEST(SortedArrayTest, NumFoundOfSortedArrayLessThanValues) {
+ std::vector<uint32_t> filter_ids = {3, 44, 51, 54, 57, 60, 121, 136, 232, 238, 278, 447, 452, 454, 455, 456, 457, 459, 463,
+ 465, 471, 472, 473, 474, 475, 478, 479, 480, 486, 490, 492, 496, 503, 508, 510, 512,
+ 515, 526, 527, 533, 534, 537, 544, 547, 551, 553, 565, 573, 574, 577, 579, 617, 621,
+ 626, 628, 635, 653, 667, 672, 675, 689, 696, 705, 711, 714, 716, 725, 731, 735, 738,
+ 739, 747, 751, 758, 762, 773, 778, 786, 787, 801, 810, 817, 821, 826, 829, 835, 836,
+ 844, 852, 853, 854, 856, 860, 861, 895, 906, 952, 953, 955, 961, 966, 968, 985, 987,
+ 988, 994, 996, 999, 1005, 1007, 1027, 1030, 1034, 1037, 1040, 1041, 1043, 1057, 1060,
+ 1062, 1065, 1073, 1095, 1119, 1127, 1136, 1137, 1144, 1148, 1150, 1158, 1161, 1167,
+ 1168, 1170, 1182, 1191, 1223, 1229, 1241, 1247, 1279, 1282, 1287, 1290, 1293, 1302,
+ 1308, 1319, 1323, 1328, 1329, 1344, 1345, 1349, 1351, 1353, 1357, 1364, 1368, 1374,
+ 1386, 1389, 1405, 1411, 1421, 1423, 1424, 1439, 1442, 1449, 1452, 1453, 1455, 1458,
+ 1496, 1500, 1501, 1508, 1512, 1526, 1533, 1541, 1546, 1551, 1568, 1579, 1582, 1588,
+ 1589, 1604, 1656, 1658, 1662, 1667, 1682, 1699, 1704, 1714, 1719, 1723, 1728, 1736,
+ 1737, 1744, 1749, 1764, 1768, 1772, 1778, 1820, 1841, 1860, 1880, 1882, 1896, 1907,
+ 1921, 1949, 1959, 1988, 1990, 1995, 2006, 2027, 2090, 2097, 2110, 2130, 2135, 2138,
+ 2144, 2154, 2159, 2165, 2177, 2186, 2204, 2229, 2234, 2255, 2272, 2301, 2319, 2360,
+ 2372, 2383, 2400, 2404, 2441, 2444, 2447, 2460, 2464, 2477, 2480, 2490, 2497, 2513,
+ 2519, 2539, 2547, 2553, 2562, 2570, 2585, 2587, 2590, 2607, 2625, 2633, 2641, 2649,
+ 2650, 2679, 2680, 2698, 2699, 2752, 2782, 2788, 2818, 2829, 2834, 2885, 2892, 2926,
+ 2948, 2954, 2958, 3071, 3088, 3094, 3099, 3124, 3148, 3149, 3151, 3152, 3197, 3212,
+ 3250, 3256, 3269};
+
+ std::vector<uint32_t> ids = {9, 19, 21, 22, 23, 25, 26, 27, 28, 29, 30, 32, 33, 34, 35, 36, 38, 39, 41, 42,
+ 46, 47, 48, 49, 52, 57, 58, 60, 61, 63, 67, 68, 69, 71, 72, 74, 75, 76, 77, 78,
+ 79, 80, 85, 86, 87, 89, 91, 93, 94, 96, 100, 102, 103, 104, 106, 109, 111, 112,
+ 113, 114, 115, 117, 118, 119, 123, 124, 127, 128, 129, 132, 133, 134, 135, 139,
+ 141, 142, 143, 144, 146, 147, 148, 149, 151, 152, 154, 155, 157, 158, 159, 161,
+ 162, 163, 164, 169, 170, 172, 174, 178, 179, 180, 181, 182, 183, 184, 185, 186,
+ 187, 188, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 206,
+ 207, 211, 212, 213, 215, 217, 219, 222, 223, 225, 226, 229, 230, 232, 233, 234,
+ 237, 239, 240, 241, 243, 244, 245, 246, 247, 248, 256, 257, 261, 262, 263, 264,
+ 265, 266, 267, 269, 270, 271, 272, 273, 274, 275, 279, 280, 281, 282, 284, 286,
+ 288, 289, 291, 292, 296, 297, 298, 299, 303, 304, 305, 307, 308, 309, 310, 311,
+ 312, 313, 314, 317, 318, 320, 321, 325, 326, 333, 337, 340, 341, 344, 345, 346,
+ 347, 350, 352, 354, 357, 359, 360, 361, 362, 363, 368, 375, 381, 383, 384, 385,
+ 386, 390, 391, 394, 395, 398, 399, 401, 404, 405, 407, 408, 409, 410, 411, 413,
+ 414, 417, 418, 419, 421, 424, 425, 427, 433, 434, 435, 437, 441, 445, 446, 1140,
+ 1495, 1590, 1646, 1707, 1872, 2201, 2844, 2866};
+
+ sorted_array arr;
+ for(auto id: ids) {
+ arr.append(id);
+ }
+
+ auto num_found = arr.numFoundOf(&filter_ids[0], filter_ids.size());
+ //ASSERT_EQ(3, num_found);
+
ids = {4,5,6,7,8};
filter_ids = {1,2,3,4,6,7,8,9,10};