Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fixed wrong `found` counts.
Kishore Nallan060959ad701a
diff --git a/TODO.md b/TODO.mdindex d06b6fd4..b005565b 100644--- a/TODO.md+++ b/TODO.md@@ -32,6 +32,8 @@- ~~Throw errors when schema is broken~~- Desc/Asc ordering with tests- Found count is wrong+- Proper pagination+- Filter query in the API- Prevent string copy during indexing- clean special chars before indexing- Minimum results should be a variable instead of blindly going with max_resultsdiff --git a/include/intersection.h b/include/array_utils.hsimilarity index 51%rename from include/intersection.hrename to include/array_utils.hindex cf7f38c2..c0bd5cc1 100644--- a/include/intersection.h+++ b/include/array_utils.h@@ -2,12 +2,15 @@#include <cstddef>#include <stdint.h>+#include <array>/* Different intersection routines adapted from:* https://github.com/lemire/SIMDCompressionAndIntersection/blob/master/src/intersection.cpp*/-class Intersection {+class ArrayUtils {public:// Fast scalar scheme designed by N. Kurz. Returns the size of out (intersected set)- static size_t scalar(const uint32_t *A, const size_t lenA, const uint32_t *B, const size_t lenB, uint32_t *out);+ static size_t and_scalar(const uint32_t *A, const size_t lenA, const uint32_t *B, const size_t lenB, uint32_t *out);++ static size_t or_scalar(const uint32_t *A, const size_t lenA, const uint32_t *B, const size_t lenB, uint32_t **out);};\ No newline at end of filediff --git a/include/collection.h b/include/collection.hindex d05a9888..e856bb60 100644--- a/include/collection.h+++ b/include/collection.h@@ -52,13 +52,14 @@ private:void search_field(std::string & query, const std::string & field, uint32_t *filter_ids, size_t filter_ids_length,std::vector<facet> & facets, const std::vector<sort_field> & sort_fields, const int num_typos,- const size_t num_results, Topster<100> &topster, size_t & num_found,- const token_ordering token_order = FREQUENCY, const bool prefix = false);+ const size_t num_results, Topster<100> &topster, uint32_t** all_result_ids,+ size_t & all_result_ids_len, const token_ordering token_order = FREQUENCY, const bool prefix = false);void search_candidates(uint32_t* filter_ids, size_t filter_ids_length, std::vector<facet> & facets,const std::vector<sort_field> & sort_fields, int & token_rank,std::vector<std::vector<art_leaf*>> & token_leaves, Topster<100> & topster,- size_t & total_results, size_t & num_found, const size_t & max_results);+ size_t & total_results, uint32_t** all_result_ids, size_t & all_result_ids_len,+ const size_t & max_results);void index_string_field(const std::string & text, const uint32_t score, art_tree *t, uint32_t seq_id,const bool verbatim) const;diff --git a/src/array_utils.cpp b/src/array_utils.cppnew file mode 100644index 00000000..69dce4ba--- /dev/null+++ b/src/array_utils.cpp@@ -0,0 +1,86 @@+#include "array_utils.h"++size_t ArrayUtils::and_scalar(const uint32_t *A, const size_t lenA,+ const uint32_t *B, const size_t lenB, uint32_t *out) {+ const uint32_t *const initout(out);+ if (lenA == 0 || lenB == 0)+ return 0;++ const uint32_t *endA = A + lenA;+ const uint32_t *endB = B + lenB;++ while (1) {+ while (*A < *B) {+ SKIP_FIRST_COMPARE:+ if (++A == endA)+ return (out - initout);+ }+ while (*A > *B) {+ if (++B == endB)+ return (out - initout);+ }+ if (*A == *B) {+ *out++ = *A;+ if (++A == endA || ++B == endB)+ return (out - initout);+ } else {+ goto SKIP_FIRST_COMPARE;+ }+ }++ return (out - initout); // NOTREACHED+}++size_t ArrayUtils::or_scalar(const uint32_t *A, const size_t lenA,+ const uint32_t *B, const size_t lenB, uint32_t **out) {+ size_t indexA = 0, indexB = 0, res_index = 0;++ if(A == nullptr) {+ *out = new uint32_t[lenB];+ memcpy(*out, B, lenB * sizeof(uint32_t));+ return lenB;+ }++ uint32_t* results = new uint32_t[lenA+lenB];++ while (indexA < lenA && indexB < lenB) {+ if (A[indexA] < B[indexB]) {+ if(res_index == 0 || results[res_index-1] != A[indexA]) {+ results[res_index] = A[indexA];+ res_index++;+ }+ indexA++;+ } else {+ if(res_index == 0 || results[res_index-1] != B[indexB]) {+ results[res_index] = B[indexB];+ res_index++;+ }+ indexB++;+ }+ }++ while (indexA < lenA) {+ if(results[res_index-1] != A[indexA]) {+ results[res_index] = A[indexA];+ res_index++;+ }++ indexA++;+ }++ while (indexB < lenB) {+ if(results[res_index-1] != B[indexB]) {+ results[res_index] = B[indexB];+ res_index++;+ }++ indexB++;+ }++ // shrink fit+ *out = new uint32_t[res_index];+ std::memmove(*out, results, res_index * sizeof(uint32_t));+ delete[] results;++ return res_index;+}\ No newline at end of filediff --git a/src/collection.cpp b/src/collection.cppindex ddce35a2..efe156ad 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -2,7 +2,7 @@#include <numeric>#include <chrono>-#include <intersection.h>+#include <array_utils.h>#include <match_score.h>#include <string_utils.h>#include <art.h>@@ -343,8 +343,9 @@ void Collection::do_facets(std::vector<facet> & facets, uint32_t* result_ids, sisize_t facet_ids_size = leaf->values->ids.getLength();uint32_t* facet_results = new uint32_t[std::min(facet_ids_size, results_size)];- const size_t facet_results_size = Intersection::scalar(result_ids, results_size,- facet_ids, facet_ids_size, facet_results);+ const size_t facet_results_size = ArrayUtils::and_scalar(result_ids, results_size,+ facet_ids, facet_ids_size,+ facet_results);const std::string facet_value((const char *)leaf->key, leaf->key_len-1); // drop trailing nulla_facet.result_map.insert(std::pair<std::string, size_t>(facet_value, facet_results_size));@@ -359,7 +360,8 @@ void Collection::do_facets(std::vector<facet> & facets, uint32_t* result_ids, sivoid Collection::search_candidates(uint32_t* filter_ids, size_t filter_ids_length, std::vector<facet> & facets,const std::vector<sort_field> & sort_fields, int & token_rank,std::vector<std::vector<art_leaf*>> & token_leaves, Topster<100> & topster,- size_t & total_results, size_t & num_found, const size_t & max_results) {+ size_t & total_results, uint32_t** all_result_ids, size_t & all_result_ids_len,+ const size_t & max_results) {const size_t combination_limit = 10;auto product = []( long long a, std::vector<art_leaf*>& b ) { return a*b.size(); };long long int N = std::accumulate(token_leaves.begin(), token_leaves.end(), 1LL, product);@@ -386,22 +388,32 @@ void Collection::search_candidates(uint32_t* filter_ids, size_t filter_ids_lengtif(filter_ids != nullptr) {// intersect once again with filter idsuint32_t* filtered_result_ids = new uint32_t[std::min(filter_ids_length, result_size)];- size_t filtered_results_size =- Intersection::scalar(filter_ids, filter_ids_length, result_ids, result_size, filtered_result_ids);+ size_t filtered_results_size = ArrayUtils::and_scalar(filter_ids, filter_ids_length, result_ids,+ result_size, filtered_result_ids);++ uint32_t* new_all_result_ids;+ all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, filtered_result_ids,+ filtered_results_size, &new_all_result_ids);+ delete [] *all_result_ids;+ *all_result_ids = new_all_result_ids;do_facets(facets, filtered_result_ids, filtered_results_size);// go through each matching document id and calculate match scorescore_results(sort_fields, token_rank, topster, query_suggestion, filtered_result_ids, filtered_results_size);- num_found += filtered_results_size;delete[] filtered_result_ids;delete[] result_ids;} else {do_facets(facets, result_ids, result_size);+ uint32_t* new_all_result_ids;+ all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, result_ids,+ result_size, &new_all_result_ids);+ delete [] *all_result_ids;+ *all_result_ids = new_all_result_ids;+score_results(sort_fields, token_rank, topster, query_suggestion, result_ids, result_size);- num_found += result_size;delete[] result_ids;}@@ -544,7 +556,8 @@ Option<uint32_t> Collection::do_filtering(uint32_t** filter_ids_out, const std::filter_ids_length = result_ids_length;} else {uint32_t* filtered_results = new uint32_t[std::min((size_t)filter_ids_length, result_ids_length)];- filter_ids_length = Intersection::scalar(filter_ids, filter_ids_length, result_ids, result_ids_length, filtered_results);+ filter_ids_length = ArrayUtils::and_scalar(filter_ids, filter_ids_length, result_ids,+ result_ids_length, filtered_results);delete [] filter_ids;delete [] result_ids;filter_ids = filtered_results;@@ -560,7 +573,6 @@ nlohmann::json Collection::search(std::string query, const std::vector<std::striconst std::string & simple_filter_query, const std::vector<std::string> & facet_fields,const std::vector<sort_field> & sort_fields, const int num_typos,const size_t num_results, const token_ordering token_order, const bool prefix) {- size_t num_found = 0;nlohmann::json result = nlohmann::json::object();std::vector<facet> facets;@@ -608,6 +620,8 @@ nlohmann::json Collection::search(std::string query, const std::vector<std::stri// Order of `fields` are used to sort resultsauto begin = std::chrono::high_resolution_clock::now();std::vector<std::pair<int, Topster<100>::KV>> field_order_kvs;+ uint32_t* all_result_ids = nullptr;+ size_t all_result_ids_len = 0;for(int i = 0; i < search_fields.size(); i++) {Topster<100> topster;@@ -615,7 +629,7 @@ nlohmann::json Collection::search(std::string query, const std::vector<std::stri// proceed to query search only when no filters are provided or when filtering produces resultsif(simple_filter_query.size() == 0 || filter_ids_length > 0) {search_field(query, field, filter_ids, filter_ids_length, facets, sort_fields, num_typos, num_results,- topster, num_found, token_order, prefix);+ topster, &all_result_ids, all_result_ids_len, token_order, prefix);topster.sort();}@@ -645,7 +659,7 @@ nlohmann::json Collection::search(std::string query, const std::vector<std::striresult["hits"].push_back(document);}- result["found"] = num_found;+ result["found"] = all_result_ids_len;result["facet_counts"] = nlohmann::json::array();@@ -682,8 +696,8 @@ nlohmann::json Collection::search(std::string query, const std::vector<std::stri*/void Collection::search_field(std::string & query, const std::string & field, uint32_t *filter_ids, size_t filter_ids_length,std::vector<facet> & facets, const std::vector<sort_field> & sort_fields, const int num_typos,- const size_t num_results, Topster<100> &topster, size_t & num_found,- const token_ordering token_order, const bool prefix) {+ const size_t num_results, Topster<100> &topster, uint32_t** all_result_ids,+ size_t & all_result_ids_len, const token_ordering token_order, const bool prefix) {std::vector<std::string> tokens;StringUtils::tokenize(query, tokens, " ", true);@@ -758,9 +772,9 @@ void Collection::search_field(std::string & query, const std::string & field, uiif(!leaves.empty()) {//!log_leaves(costs[token_index], token, leaves);token_leaves.push_back(leaves);- token_to_count[token] = leaves.at(0)->values->ids.getLength();+ token_to_count[token] = std::max(token_to_count[token], leaves.at(0)->values->ids.getLength());} else {- // No result at `cost = costs[token_index]` => remove cost for token and re-do combinations+ // No result at `cost = costs[token_index]`. Remove costs until `cost` for token and re-do combinationsauto it = std::find(token_to_costs[token_index].begin(), token_to_costs[token_index].end(), costs[token_index]);if(it != token_to_costs[token_index].end()) {token_to_costs[token_index].erase(it);@@ -773,9 +787,9 @@ void Collection::search_field(std::string & query, const std::string & field, ui}}+ // To continue outerloop on new cost combinationn = -1;N = std::accumulate(token_to_costs.begin(), token_to_costs.end(), 1LL, product);-break;}@@ -785,7 +799,7 @@ void Collection::search_field(std::string & query, const std::string & field, uiif(token_leaves.size() != 0 && token_leaves.size() == tokens.size()) {// If all tokens were found, go ahead and search for candidates with what we have so farsearch_candidates(filter_ids, filter_ids_length, facets, sort_fields, token_rank, token_leaves, topster,- total_results, num_found, max_results);+ total_results, all_result_ids, all_result_ids_len, max_results);if (total_results >= max_results) {// If we don't find enough results, we continue outerloop (looking at tokens with greater cost)@@ -819,7 +833,7 @@ void Collection::search_field(std::string & query, const std::string & field, ui}return search_field(truncated_query, field, filter_ids, filter_ids_length, facets, sort_fields, num_typos,- num_results, topster, num_found, token_order, prefix);+ num_results, topster, all_result_ids, all_result_ids_len, token_order, prefix);}}diff --git a/src/intersection.cpp b/src/intersection.cppdeleted file mode 100644index 3dd9e95a..00000000--- a/src/intersection.cpp+++ /dev/null@@ -1,32 +0,0 @@-#include "intersection.h"--size_t Intersection::scalar(const uint32_t *A, const size_t lenA,- const uint32_t *B, const size_t lenB, uint32_t *out) {- const uint32_t *const initout(out);- if (lenA == 0 || lenB == 0)- return 0;-- const uint32_t *endA = A + lenA;- const uint32_t *endB = B + lenB;-- while (1) {- while (*A < *B) {- SKIP_FIRST_COMPARE:- if (++A == endA)- return (out - initout);- }- while (*A > *B) {- if (++B == endB)- return (out - initout);- }- if (*A == *B) {- *out++ = *A;- if (++A == endA || ++B == endB)- return (out - initout);- } else {- goto SKIP_FIRST_COMPARE;- }- }-- return (out - initout); // NOTREACHED-}diff --git a/src/sorted_array.cpp b/src/sorted_array.cppindex b02ccd1c..9f32aea5 100644--- a/src/sorted_array.cpp+++ b/src/sorted_array.cpp@@ -1,5 +1,5 @@#include "sorted_array.h"-#include "intersection.h"+#include "array_utils.h"void sorted_array::load(const uint32_t *sorted_array, const uint32_t array_length) {min = sorted_array[0];@@ -173,7 +173,7 @@ size_t sorted_array::intersect(uint32_t* arr, const size_t arr_length, uint32_t*uint32_t* curr = uncompress();uint32_t* results = new uint32_t[std::min(arr_length, (size_t) length)];- size_t results_length = Intersection::scalar(arr, arr_length, curr, length, results);+ size_t results_length = ArrayUtils::and_scalar(arr, arr_length, curr, length, results);delete[] curr;*results_out = results;diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 539d5d76..56a91593 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -79,6 +79,7 @@ TEST_F(CollectionTest, ExactPhraseSearch) {std::vector<std::string> facets;nlohmann::json results = collection->search("rocket launch", query_fields, "", facets, sort_fields, 0, 10);ASSERT_EQ(5, results["hits"].size());+ ASSERT_EQ(5, results["found"].get<uint32_t>());/*Sort by (match, diff, score)@@ -101,6 +102,8 @@ TEST_F(CollectionTest, ExactPhraseSearch) {// Check paginationresults = collection->search("rocket launch", query_fields, "", facets, sort_fields, 0, 3);ASSERT_EQ(3, results["hits"].size());+ ASSERT_EQ(4, results["found"].get<uint32_t>());+for(size_t i = 0; i < 3; i++) {nlohmann::json result = results["hits"].at(i);std::string id = ids.at(i);@@ -264,6 +267,7 @@ TEST_F(CollectionTest, TextContainingAnActualTypo) {std::vector<std::string> facets;nlohmann::json results = collection->search("ISX what", query_fields, "", facets, sort_fields, 1, 4, FREQUENCY, false);ASSERT_EQ(4, results["hits"].size());+ ASSERT_EQ(4, results["found"].get<uint32_t>());std::vector<std::string> ids = {"19", "6", "21", "8"};@@ -277,6 +281,7 @@ TEST_F(CollectionTest, TextContainingAnActualTypo) {// Record containing exact token match should appear firstresults = collection->search("ISX", query_fields, "", facets, sort_fields, 1, 10, FREQUENCY, false);ASSERT_EQ(8, results["hits"].size());+ ASSERT_EQ(8, results["found"].get<uint32_t>());ids = {"20", "19", "6", "3", "21", "4", "10", "8"};