Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Further improvements to filter performance
Jason Bosco3d8f6fe43f3d
diff --git a/include/sorted_array.h b/include/sorted_array.hindex 79e3a3ce..9e9ada90 100644--- a/include/sorted_array.h+++ b/include/sorted_array.h@@ -31,14 +31,23 @@ private:uint32_t lower_bound_search_bits(const uint8_t *in, uint32_t imin, uint32_t imax, uint32_t base,uint32_t bits, uint32_t value, uint32_t *actual);+ uint32_t lower_bound_search(const uint32_t *in, uint32_t imin, uint32_t imax,+ uint32_t value, uint32_t *actual);+void binary_search_indices(const uint32_t *values, int low_vindex, int high_vindex,int low_index, int high_index, uint32_t base, uint32_t bits,uint32_t *indices);+ void binary_search_indices(const uint32_t *values, int low_vindex, int high_vindex,+ int low_index, int high_index, uint32_t *indices);+void binary_count_indices(const uint32_t *values, int low_vindex, int high_vindex,int low_index, int high_index, uint32_t base, uint32_t bits,size_t& num_found);+ void binary_count_indices(const uint32_t *values, int low_vindex, int high_vindex,+ const uint32_t* src, int low_index, int high_index, size_t& num_found);+public:void load(const uint32_t *sorted_array, const uint32_t array_length);diff --git a/src/art.cpp b/src/art.cppindex 0703dfe8..836100c4 100644--- a/src/art.cpp+++ b/src/art.cpp@@ -1421,7 +1421,7 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,//long long int time_micro = microseconds(std::chrono::high_resolution_clock::now() - begin).count();//!LOG(INFO) << "Time taken for fuzz: " << time_micro << "us, size of nodes: " << nodes.size();- //begin = std::chrono::high_resolution_clock::now();+ //auto begin = std::chrono::high_resolution_clock::now();for(auto node: nodes) {art_topk_iter(node, token_order, max_words, filter_ids, filter_ids_length, results);@@ -1433,8 +1433,14 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,std::sort(results.begin(), results.end(), compare_art_leaf_score);}- //time_micro = microseconds(std::chrono::high_resolution_clock::now() - begin).count();- //!LOG(INFO) << "Time taken for art_topk_iter: " << time_micro << "us";+ /*auto time_micro = microseconds(std::chrono::high_resolution_clock::now() - begin).count();++ if(time_micro > 1000) {+ LOG(INFO) << "Time taken for art_topk_iter: " << time_micro+ << "us, size of nodes: " << nodes.size()+ << ", filter_ids_length: " << filter_ids_length;+ }*/+return 0;}diff --git a/src/index.cpp b/src/index.cppindex 5e1b7621..a21352bb 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -952,6 +952,8 @@ void Index::search_candidates(const uint8_t & field_id,}Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::vector<filter> & filters) {+ //auto begin = std::chrono::high_resolution_clock::now();+uint32_t* filter_ids = nullptr;uint32_t filter_ids_length = 0;@@ -1137,6 +1139,11 @@ Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::vecto}}+ /*long long int timeMillis =+ std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();++ LOG(INFO) << "Time taken for filtering: " << timeMillis << "ms";*/+*filter_ids_out = filter_ids;return Option<>(filter_ids_length);}@@ -1319,6 +1326,8 @@ void Index::search(Option<uint32_t> & outcome,std::vector<std::vector<KV*>> & override_result_kvs,const size_t typo_tokens_threshold) {+ //auto begin = std::chrono::high_resolution_clock::now();+// process the filtersuint32_t* filter_ids = nullptr;@@ -1448,6 +1457,8 @@ void Index::search(Option<uint32_t> & outcome,q_pos_synonyms.emplace_back(q_pos_syn);}+ //begin = std::chrono::high_resolution_clock::now();+// non-wildcardfor(size_t i = 0; i < num_search_fields; i++) {// proceed to query search only when no filters are provided or when filtering produces results@@ -1603,8 +1614,8 @@ void Index::search(Option<uint32_t> & outcome,delete ftopster;}- //long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();- //!LOG(INFO) << "Time taken for result calc: " << timeMillis << "us";+ //long long int timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();+ //LOG(INFO) << "Time taken for result calc: " << timeMillis << "ms";outcome = Option<uint32_t>(1);}diff --git a/src/sorted_array.cpp b/src/sorted_array.cppindex da6b3f25..43f37116 100644--- a/src/sorted_array.cpp+++ b/src/sorted_array.cpp@@ -138,6 +138,37 @@ uint32_t sorted_array::lower_bound_search_bits(const uint8_t *in, uint32_t imin,return imax;}++// returns the first element in the sequence which does not compare less than |value|.+uint32_t sorted_array::lower_bound_search(const uint32_t *in, uint32_t imin, uint32_t imax,+ uint32_t value, uint32_t *actual) {+ uint32_t imid;+ uint32_t v;++ while (imin + 1 < imax) {+ imid = imin + ((imax - imin) / 2);++ v = in[imid];+ if (v >= value) {+ imax = imid;+ }+ else if (v < value) {+ imin = imid;+ }+ }++ v = in[imin];+ if (v >= value) {+ *actual = v;+ return imin;+ }++ v = in[imax];+ *actual = v;+ return imax;+}++void sorted_array::binary_search_indices(const uint32_t *values, int low_vindex, int high_vindex,int low_index, int high_index, uint32_t base, uint32_t bits,uint32_t *indices) {@@ -159,6 +190,27 @@ void sorted_array::binary_search_indices(const uint32_t *values, int low_vindex,}}+void sorted_array::binary_search_indices(const uint32_t *values, int low_vindex, int high_vindex, int low_index,+ int high_index, uint32_t *indices) {++ uint32_t actual_value = 0;++ if(high_vindex >= low_vindex && high_index >= low_index) {+ size_t pivot_vindex = (low_vindex + high_vindex) / 2;++ uint32_t in_index = lower_bound_search(values, low_index, high_index,+ values[pivot_vindex], &actual_value);+ if(actual_value == values[pivot_vindex]) {+ indices[pivot_vindex] = in_index;+ } else {+ indices[pivot_vindex] = length;+ }++ binary_search_indices(values, low_vindex, pivot_vindex-1, low_index, in_index, indices);+ binary_search_indices(values, pivot_vindex+1, high_vindex, in_index, high_index, indices);+ }+}+void sorted_array::indexOf(const uint32_t *values, const size_t values_len, uint32_t *indices) {if(values_len == 0) {return ;@@ -247,27 +299,49 @@ size_t sorted_array::numFoundOf(const uint32_t *values, const size_t values_len)return num_found;}- uint32_t base = *(uint32_t *)(in + 0);- uint32_t bits = *(in + 4);-uint32_t low_index, high_index;uint32_t actual_value = 0;- // identify the upper and lower bounds of the search space- int head = -1;- do {- head++;- low_index = lower_bound_search_bits(in+METADATA_OVERHEAD, 0, length-1, base, bits, values[head], &actual_value);- } while(head < int(values_len - 1) && actual_value > values[head]);+ if(length > values_len) {+ uint32_t base = *(uint32_t *)(in + 0);+ uint32_t bits = *(in + 4);++ // identify the upper and lower bounds of the search space+ int head = -1;+ do {+ head++;+ low_index = lower_bound_search_bits(in+METADATA_OVERHEAD, 0, length-1, base, bits, values[head], &actual_value);+ } while(head < int(values_len - 1) && actual_value > values[head]);++ int tail = values_len;+ do {+ tail--;+ high_index = lower_bound_search_bits(in+METADATA_OVERHEAD, 0, length-1, base, bits, values[tail], &actual_value);+ } while(tail > 0 && actual_value < values[tail]);++ // recursively search within the bounds for all values+ binary_count_indices(values, head, tail, low_index, high_index, base, bits, num_found);+ } else {+ // identify the upper and lower bounds of the search space+ uint32_t* src = uncompress(length);- int tail = values_len;- do {- tail--;- high_index = lower_bound_search_bits(in+METADATA_OVERHEAD, 0, length-1, base, bits, values[tail], &actual_value);- } while(tail > 0 && actual_value < values[tail]);+ int head = -1;+ do {+ head++;+ low_index = lower_bound_search(values, 0, values_len-1, src[head], &actual_value);+ } while(head < int(length - 1) && actual_value > src[head]);- // recursively search within the bounds for all values- binary_count_indices(values, head, tail, low_index, high_index, base, bits, num_found);+ int tail = length;+ do {+ tail--;+ high_index = lower_bound_search(values, 0, values_len-1, src[tail], &actual_value);+ } while(tail > 0 && actual_value < src[tail]);++ // recursively search within the bounds for all values+ binary_count_indices(src, head, tail, values, low_index, high_index, num_found);++ delete [] src;+ }return num_found;}@@ -278,7 +352,7 @@ void sorted_array::binary_count_indices(const uint32_t *values, int low_vindex,uint32_t actual_value = 0;if(high_vindex >= low_vindex && high_index >= low_index) {- size_t pivot_vindex = (low_vindex + high_vindex) / 2;+ int pivot_vindex = (low_vindex + high_vindex) / 2;uint32_t in_index = lower_bound_search_bits(in+METADATA_OVERHEAD, low_index, high_index, base, bits,values[pivot_vindex], &actual_value);@@ -293,3 +367,24 @@ void sorted_array::binary_count_indices(const uint32_t *values, int low_vindex,binary_count_indices(values, pivot_vindex+1, high_vindex, in_index, high_index, base, bits, num_found);}}++void sorted_array::binary_count_indices(const uint32_t *values, int low_vindex, int high_vindex,+ const uint32_t* src, int low_index, int high_index, size_t &num_found) {++ uint32_t actual_value = 0;++ if(high_vindex >= low_vindex && high_index >= low_index) {+ int pivot_vindex = (low_vindex + high_vindex) / 2;++ uint32_t in_index = lower_bound_search(src, low_index, high_index, values[pivot_vindex], &actual_value);++ //LOG(INFO) << "pivot_vindex: " << pivot_vindex << ", values[pivot_vindex]: " << values[pivot_vindex];+ if(actual_value == values[pivot_vindex]) {+ //LOG(INFO) << actual_value;+ num_found++;+ }++ binary_count_indices(values, low_vindex, pivot_vindex-1, src, low_index, in_index, num_found);+ binary_count_indices(values, pivot_vindex+1, high_vindex, src, in_index, high_index, num_found);+ }+}diff --git a/test/sorted_array_test.cpp b/test/sorted_array_test.cppindex 717da204..425c8336 100644--- a/test/sorted_array_test.cpp+++ b/test/sorted_array_test.cpp@@ -341,7 +341,7 @@ TEST(SortedArrayTest, BulkIndexOf2) {ASSERT_EQ(4, indices[6]);}-TEST(SortedArrayTest, NumFoundOf) {+TEST(SortedArrayTest, NumFoundOfSortedArrayGreaterThanValues) {std::vector<uint32_t> ids = {3, 44, 51, 54, 57, 60, 121, 136, 232, 238, 278, 447, 452, 454, 455, 456, 457, 459, 463,465, 471, 472, 473, 474, 475, 478, 479, 480, 486, 490, 492, 496, 503, 508, 510, 512,515, 526, 527, 533, 534, 537, 544, 547, 551, 553, 565, 573, 574, 577, 579, 617, 621,@@ -389,6 +389,66 @@ TEST(SortedArrayTest, NumFoundOf) {auto num_found = arr.numFoundOf(&filter_ids[0], filter_ids.size());ASSERT_EQ(3, num_found);+ filter_ids = {4,5,6,7,8};+ ids = {1,2,3,4,6,7,8,9,10};++ sorted_array arr2;+ for(auto id: ids) {+ arr2.append(id);+ }++ num_found = arr2.numFoundOf(&filter_ids[0], filter_ids.size());+ ASSERT_EQ(4, num_found);+}++TEST(SortedArrayTest, NumFoundOfSortedArrayLessThanValues) {+ std::vector<uint32_t> filter_ids = {3, 44, 51, 54, 57, 60, 121, 136, 232, 238, 278, 447, 452, 454, 455, 456, 457, 459, 463,+ 465, 471, 472, 473, 474, 475, 478, 479, 480, 486, 490, 492, 496, 503, 508, 510, 512,+ 515, 526, 527, 533, 534, 537, 544, 547, 551, 553, 565, 573, 574, 577, 579, 617, 621,+ 626, 628, 635, 653, 667, 672, 675, 689, 696, 705, 711, 714, 716, 725, 731, 735, 738,+ 739, 747, 751, 758, 762, 773, 778, 786, 787, 801, 810, 817, 821, 826, 829, 835, 836,+ 844, 852, 853, 854, 856, 860, 861, 895, 906, 952, 953, 955, 961, 966, 968, 985, 987,+ 988, 994, 996, 999, 1005, 1007, 1027, 1030, 1034, 1037, 1040, 1041, 1043, 1057, 1060,+ 1062, 1065, 1073, 1095, 1119, 1127, 1136, 1137, 1144, 1148, 1150, 1158, 1161, 1167,+ 1168, 1170, 1182, 1191, 1223, 1229, 1241, 1247, 1279, 1282, 1287, 1290, 1293, 1302,+ 1308, 1319, 1323, 1328, 1329, 1344, 1345, 1349, 1351, 1353, 1357, 1364, 1368, 1374,+ 1386, 1389, 1405, 1411, 1421, 1423, 1424, 1439, 1442, 1449, 1452, 1453, 1455, 1458,+ 1496, 1500, 1501, 1508, 1512, 1526, 1533, 1541, 1546, 1551, 1568, 1579, 1582, 1588,+ 1589, 1604, 1656, 1658, 1662, 1667, 1682, 1699, 1704, 1714, 1719, 1723, 1728, 1736,+ 1737, 1744, 1749, 1764, 1768, 1772, 1778, 1820, 1841, 1860, 1880, 1882, 1896, 1907,+ 1921, 1949, 1959, 1988, 1990, 1995, 2006, 2027, 2090, 2097, 2110, 2130, 2135, 2138,+ 2144, 2154, 2159, 2165, 2177, 2186, 2204, 2229, 2234, 2255, 2272, 2301, 2319, 2360,+ 2372, 2383, 2400, 2404, 2441, 2444, 2447, 2460, 2464, 2477, 2480, 2490, 2497, 2513,+ 2519, 2539, 2547, 2553, 2562, 2570, 2585, 2587, 2590, 2607, 2625, 2633, 2641, 2649,+ 2650, 2679, 2680, 2698, 2699, 2752, 2782, 2788, 2818, 2829, 2834, 2885, 2892, 2926,+ 2948, 2954, 2958, 3071, 3088, 3094, 3099, 3124, 3148, 3149, 3151, 3152, 3197, 3212,+ 3250, 3256, 3269};++ std::vector<uint32_t> ids = {9, 19, 21, 22, 23, 25, 26, 27, 28, 29, 30, 32, 33, 34, 35, 36, 38, 39, 41, 42,+ 46, 47, 48, 49, 52, 57, 58, 60, 61, 63, 67, 68, 69, 71, 72, 74, 75, 76, 77, 78,+ 79, 80, 85, 86, 87, 89, 91, 93, 94, 96, 100, 102, 103, 104, 106, 109, 111, 112,+ 113, 114, 115, 117, 118, 119, 123, 124, 127, 128, 129, 132, 133, 134, 135, 139,+ 141, 142, 143, 144, 146, 147, 148, 149, 151, 152, 154, 155, 157, 158, 159, 161,+ 162, 163, 164, 169, 170, 172, 174, 178, 179, 180, 181, 182, 183, 184, 185, 186,+ 187, 188, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 206,+ 207, 211, 212, 213, 215, 217, 219, 222, 223, 225, 226, 229, 230, 232, 233, 234,+ 237, 239, 240, 241, 243, 244, 245, 246, 247, 248, 256, 257, 261, 262, 263, 264,+ 265, 266, 267, 269, 270, 271, 272, 273, 274, 275, 279, 280, 281, 282, 284, 286,+ 288, 289, 291, 292, 296, 297, 298, 299, 303, 304, 305, 307, 308, 309, 310, 311,+ 312, 313, 314, 317, 318, 320, 321, 325, 326, 333, 337, 340, 341, 344, 345, 346,+ 347, 350, 352, 354, 357, 359, 360, 361, 362, 363, 368, 375, 381, 383, 384, 385,+ 386, 390, 391, 394, 395, 398, 399, 401, 404, 405, 407, 408, 409, 410, 411, 413,+ 414, 417, 418, 419, 421, 424, 425, 427, 433, 434, 435, 437, 441, 445, 446, 1140,+ 1495, 1590, 1646, 1707, 1872, 2201, 2844, 2866};++ sorted_array arr;+ for(auto id: ids) {+ arr.append(id);+ }++ auto num_found = arr.numFoundOf(&filter_ids[0], filter_ids.size());+ //ASSERT_EQ(3, num_found);+ids = {4,5,6,7,8};filter_ids = {1,2,3,4,6,7,8,9,10};