Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fixed wrong `found` counts.

Kishore Nallan060959ad701a
diff --git a/TODO.md b/TODO.md
index d06b6fd4..b005565b 100644
--- a/TODO.md
+++ b/TODO.md
@@ -32,6 +32,8 @@
- ~~Throw errors when schema is broken~~
- Desc/Asc ordering with tests
- Found count is wrong
+- Proper pagination
+- Filter query in the API
- Prevent string copy during indexing
- clean special chars before indexing
- Minimum results should be a variable instead of blindly going with max_results
diff --git a/include/intersection.h b/include/array_utils.h
similarity index 51%
rename from include/intersection.h
rename to include/array_utils.h
index cf7f38c2..c0bd5cc1 100644
--- a/include/intersection.h
+++ b/include/array_utils.h
@@ -2,12 +2,15 @@
#include <cstddef>
#include <stdint.h>
+#include <array>
/* Different intersection routines adapted from:
* https://github.com/lemire/SIMDCompressionAndIntersection/blob/master/src/intersection.cpp
*/
-class Intersection {
+class ArrayUtils {
public:
// Fast scalar scheme designed by N. Kurz. Returns the size of out (intersected set)
- static size_t scalar(const uint32_t *A, const size_t lenA, const uint32_t *B, const size_t lenB, uint32_t *out);
+ static size_t and_scalar(const uint32_t *A, const size_t lenA, const uint32_t *B, const size_t lenB, uint32_t *out);
+
+ static size_t or_scalar(const uint32_t *A, const size_t lenA, const uint32_t *B, const size_t lenB, uint32_t **out);
};
\ No newline at end of file
diff --git a/include/collection.h b/include/collection.h
index d05a9888..e856bb60 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -52,13 +52,14 @@ private:
void search_field(std::string & query, const std::string & field, uint32_t *filter_ids, size_t filter_ids_length,
std::vector<facet> & facets, const std::vector<sort_field> & sort_fields, const int num_typos,
- const size_t num_results, Topster<100> &topster, size_t & num_found,
- const token_ordering token_order = FREQUENCY, const bool prefix = false);
+ const size_t num_results, Topster<100> &topster, uint32_t** all_result_ids,
+ size_t & all_result_ids_len, const token_ordering token_order = FREQUENCY, const bool prefix = false);
void search_candidates(uint32_t* filter_ids, size_t filter_ids_length, std::vector<facet> & facets,
const std::vector<sort_field> & sort_fields, int & token_rank,
std::vector<std::vector<art_leaf*>> & token_leaves, Topster<100> & topster,
- size_t & total_results, size_t & num_found, const size_t & max_results);
+ size_t & total_results, uint32_t** all_result_ids, size_t & all_result_ids_len,
+ const size_t & max_results);
void index_string_field(const std::string & text, const uint32_t score, art_tree *t, uint32_t seq_id,
const bool verbatim) const;
diff --git a/src/array_utils.cpp b/src/array_utils.cpp
new file mode 100644
index 00000000..69dce4ba
--- /dev/null
+++ b/src/array_utils.cpp
@@ -0,0 +1,86 @@
+#include "array_utils.h"
+
+size_t ArrayUtils::and_scalar(const uint32_t *A, const size_t lenA,
+ const uint32_t *B, const size_t lenB, uint32_t *out) {
+ const uint32_t *const initout(out);
+ if (lenA == 0 || lenB == 0)
+ return 0;
+
+ const uint32_t *endA = A + lenA;
+ const uint32_t *endB = B + lenB;
+
+ while (1) {
+ while (*A < *B) {
+ SKIP_FIRST_COMPARE:
+ if (++A == endA)
+ return (out - initout);
+ }
+ while (*A > *B) {
+ if (++B == endB)
+ return (out - initout);
+ }
+ if (*A == *B) {
+ *out++ = *A;
+ if (++A == endA || ++B == endB)
+ return (out - initout);
+ } else {
+ goto SKIP_FIRST_COMPARE;
+ }
+ }
+
+ return (out - initout); // NOTREACHED
+}
+
+size_t ArrayUtils::or_scalar(const uint32_t *A, const size_t lenA,
+ const uint32_t *B, const size_t lenB, uint32_t **out) {
+ size_t indexA = 0, indexB = 0, res_index = 0;
+
+ if(A == nullptr) {
+ *out = new uint32_t[lenB];
+ memcpy(*out, B, lenB * sizeof(uint32_t));
+ return lenB;
+ }
+
+ uint32_t* results = new uint32_t[lenA+lenB];
+
+ while (indexA < lenA && indexB < lenB) {
+ if (A[indexA] < B[indexB]) {
+ if(res_index == 0 || results[res_index-1] != A[indexA]) {
+ results[res_index] = A[indexA];
+ res_index++;
+ }
+ indexA++;
+ } else {
+ if(res_index == 0 || results[res_index-1] != B[indexB]) {
+ results[res_index] = B[indexB];
+ res_index++;
+ }
+ indexB++;
+ }
+ }
+
+ while (indexA < lenA) {
+ if(results[res_index-1] != A[indexA]) {
+ results[res_index] = A[indexA];
+ res_index++;
+ }
+
+ indexA++;
+ }
+
+ while (indexB < lenB) {
+ if(results[res_index-1] != B[indexB]) {
+ results[res_index] = B[indexB];
+ res_index++;
+ }
+
+ indexB++;
+ }
+
+ // shrink fit
+ *out = new uint32_t[res_index];
+ std::memmove(*out, results, res_index * sizeof(uint32_t));
+ delete[] results;
+
+ return res_index;
+}
\ No newline at end of file
diff --git a/src/collection.cpp b/src/collection.cpp
index ddce35a2..efe156ad 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -2,7 +2,7 @@
#include <numeric>
#include <chrono>
-#include <intersection.h>
+#include <array_utils.h>
#include <match_score.h>
#include <string_utils.h>
#include <art.h>
@@ -343,8 +343,9 @@ void Collection::do_facets(std::vector<facet> & facets, uint32_t* result_ids, si
size_t facet_ids_size = leaf->values->ids.getLength();
uint32_t* facet_results = new uint32_t[std::min(facet_ids_size, results_size)];
- const size_t facet_results_size = Intersection::scalar(result_ids, results_size,
- facet_ids, facet_ids_size, facet_results);
+ const size_t facet_results_size = ArrayUtils::and_scalar(result_ids, results_size,
+ facet_ids, facet_ids_size,
+ facet_results);
const std::string facet_value((const char *)leaf->key, leaf->key_len-1); // drop trailing null
a_facet.result_map.insert(std::pair<std::string, size_t>(facet_value, facet_results_size));
@@ -359,7 +360,8 @@ void Collection::do_facets(std::vector<facet> & facets, uint32_t* result_ids, si
void Collection::search_candidates(uint32_t* filter_ids, size_t filter_ids_length, std::vector<facet> & facets,
const std::vector<sort_field> & sort_fields, int & token_rank,
std::vector<std::vector<art_leaf*>> & token_leaves, Topster<100> & topster,
- size_t & total_results, size_t & num_found, const size_t & max_results) {
+ size_t & total_results, uint32_t** all_result_ids, size_t & all_result_ids_len,
+ const size_t & max_results) {
const size_t combination_limit = 10;
auto product = []( long long a, std::vector<art_leaf*>& b ) { return a*b.size(); };
long long int N = std::accumulate(token_leaves.begin(), token_leaves.end(), 1LL, product);
@@ -386,22 +388,32 @@ void Collection::search_candidates(uint32_t* filter_ids, size_t filter_ids_lengt
if(filter_ids != nullptr) {
// intersect once again with filter ids
uint32_t* filtered_result_ids = new uint32_t[std::min(filter_ids_length, result_size)];
- size_t filtered_results_size =
- Intersection::scalar(filter_ids, filter_ids_length, result_ids, result_size, filtered_result_ids);
+ size_t filtered_results_size = ArrayUtils::and_scalar(filter_ids, filter_ids_length, result_ids,
+ result_size, filtered_result_ids);
+
+ uint32_t* new_all_result_ids;
+ all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, filtered_result_ids,
+ filtered_results_size, &new_all_result_ids);
+ delete [] *all_result_ids;
+ *all_result_ids = new_all_result_ids;
do_facets(facets, filtered_result_ids, filtered_results_size);
// go through each matching document id and calculate match score
score_results(sort_fields, token_rank, topster, query_suggestion, filtered_result_ids, filtered_results_size);
- num_found += filtered_results_size;
delete[] filtered_result_ids;
delete[] result_ids;
} else {
do_facets(facets, result_ids, result_size);
+ uint32_t* new_all_result_ids;
+ all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, result_ids,
+ result_size, &new_all_result_ids);
+ delete [] *all_result_ids;
+ *all_result_ids = new_all_result_ids;
+
score_results(sort_fields, token_rank, topster, query_suggestion, result_ids, result_size);
- num_found += result_size;
delete[] result_ids;
}
@@ -544,7 +556,8 @@ Option<uint32_t> Collection::do_filtering(uint32_t** filter_ids_out, const std::
filter_ids_length = result_ids_length;
} else {
uint32_t* filtered_results = new uint32_t[std::min((size_t)filter_ids_length, result_ids_length)];
- filter_ids_length = Intersection::scalar(filter_ids, filter_ids_length, result_ids, result_ids_length, filtered_results);
+ filter_ids_length = ArrayUtils::and_scalar(filter_ids, filter_ids_length, result_ids,
+ result_ids_length, filtered_results);
delete [] filter_ids;
delete [] result_ids;
filter_ids = filtered_results;
@@ -560,7 +573,6 @@ nlohmann::json Collection::search(std::string query, const std::vector<std::stri
const std::string & simple_filter_query, const std::vector<std::string> & facet_fields,
const std::vector<sort_field> & sort_fields, const int num_typos,
const size_t num_results, const token_ordering token_order, const bool prefix) {
- size_t num_found = 0;
nlohmann::json result = nlohmann::json::object();
std::vector<facet> facets;
@@ -608,6 +620,8 @@ nlohmann::json Collection::search(std::string query, const std::vector<std::stri
// Order of `fields` are used to sort results
auto begin = std::chrono::high_resolution_clock::now();
std::vector<std::pair<int, Topster<100>::KV>> field_order_kvs;
+ uint32_t* all_result_ids = nullptr;
+ size_t all_result_ids_len = 0;
for(int i = 0; i < search_fields.size(); i++) {
Topster<100> topster;
@@ -615,7 +629,7 @@ nlohmann::json Collection::search(std::string query, const std::vector<std::stri
// proceed to query search only when no filters are provided or when filtering produces results
if(simple_filter_query.size() == 0 || filter_ids_length > 0) {
search_field(query, field, filter_ids, filter_ids_length, facets, sort_fields, num_typos, num_results,
- topster, num_found, token_order, prefix);
+ topster, &all_result_ids, all_result_ids_len, token_order, prefix);
topster.sort();
}
@@ -645,7 +659,7 @@ nlohmann::json Collection::search(std::string query, const std::vector<std::stri
result["hits"].push_back(document);
}
- result["found"] = num_found;
+ result["found"] = all_result_ids_len;
result["facet_counts"] = nlohmann::json::array();
@@ -682,8 +696,8 @@ nlohmann::json Collection::search(std::string query, const std::vector<std::stri
*/
void Collection::search_field(std::string & query, const std::string & field, uint32_t *filter_ids, size_t filter_ids_length,
std::vector<facet> & facets, const std::vector<sort_field> & sort_fields, const int num_typos,
- const size_t num_results, Topster<100> &topster, size_t & num_found,
- const token_ordering token_order, const bool prefix) {
+ const size_t num_results, Topster<100> &topster, uint32_t** all_result_ids,
+ size_t & all_result_ids_len, const token_ordering token_order, const bool prefix) {
std::vector<std::string> tokens;
StringUtils::tokenize(query, tokens, " ", true);
@@ -758,9 +772,9 @@ void Collection::search_field(std::string & query, const std::string & field, ui
if(!leaves.empty()) {
//!log_leaves(costs[token_index], token, leaves);
token_leaves.push_back(leaves);
- token_to_count[token] = leaves.at(0)->values->ids.getLength();
+ token_to_count[token] = std::max(token_to_count[token], leaves.at(0)->values->ids.getLength());
} else {
- // No result at `cost = costs[token_index]` => remove cost for token and re-do combinations
+ // No result at `cost = costs[token_index]`. Remove costs until `cost` for token and re-do combinations
auto it = std::find(token_to_costs[token_index].begin(), token_to_costs[token_index].end(), costs[token_index]);
if(it != token_to_costs[token_index].end()) {
token_to_costs[token_index].erase(it);
@@ -773,9 +787,9 @@ void Collection::search_field(std::string & query, const std::string & field, ui
}
}
+ // To continue outerloop on new cost combination
n = -1;
N = std::accumulate(token_to_costs.begin(), token_to_costs.end(), 1LL, product);
-
break;
}
@@ -785,7 +799,7 @@ void Collection::search_field(std::string & query, const std::string & field, ui
if(token_leaves.size() != 0 && token_leaves.size() == tokens.size()) {
// If all tokens were found, go ahead and search for candidates with what we have so far
search_candidates(filter_ids, filter_ids_length, facets, sort_fields, token_rank, token_leaves, topster,
- total_results, num_found, max_results);
+ total_results, all_result_ids, all_result_ids_len, max_results);
if (total_results >= max_results) {
// If we don't find enough results, we continue outerloop (looking at tokens with greater cost)
@@ -819,7 +833,7 @@ void Collection::search_field(std::string & query, const std::string & field, ui
}
return search_field(truncated_query, field, filter_ids, filter_ids_length, facets, sort_fields, num_typos,
- num_results, topster, num_found, token_order, prefix);
+ num_results, topster, all_result_ids, all_result_ids_len, token_order, prefix);
}
}
diff --git a/src/intersection.cpp b/src/intersection.cpp
deleted file mode 100644
index 3dd9e95a..00000000
--- a/src/intersection.cpp
+++ /dev/null
@@ -1,32 +0,0 @@
-#include "intersection.h"
-
-size_t Intersection::scalar(const uint32_t *A, const size_t lenA,
- const uint32_t *B, const size_t lenB, uint32_t *out) {
- const uint32_t *const initout(out);
- if (lenA == 0 || lenB == 0)
- return 0;
-
- const uint32_t *endA = A + lenA;
- const uint32_t *endB = B + lenB;
-
- while (1) {
- while (*A < *B) {
- SKIP_FIRST_COMPARE:
- if (++A == endA)
- return (out - initout);
- }
- while (*A > *B) {
- if (++B == endB)
- return (out - initout);
- }
- if (*A == *B) {
- *out++ = *A;
- if (++A == endA || ++B == endB)
- return (out - initout);
- } else {
- goto SKIP_FIRST_COMPARE;
- }
- }
-
- return (out - initout); // NOTREACHED
-}
diff --git a/src/sorted_array.cpp b/src/sorted_array.cpp
index b02ccd1c..9f32aea5 100644
--- a/src/sorted_array.cpp
+++ b/src/sorted_array.cpp
@@ -1,5 +1,5 @@
#include "sorted_array.h"
-#include "intersection.h"
+#include "array_utils.h"
void sorted_array::load(const uint32_t *sorted_array, const uint32_t array_length) {
min = sorted_array[0];
@@ -173,7 +173,7 @@ size_t sorted_array::intersect(uint32_t* arr, const size_t arr_length, uint32_t*
uint32_t* curr = uncompress();
uint32_t* results = new uint32_t[std::min(arr_length, (size_t) length)];
- size_t results_length = Intersection::scalar(arr, arr_length, curr, length, results);
+ size_t results_length = ArrayUtils::and_scalar(arr, arr_length, curr, length, results);
delete[] curr;
*results_out = results;
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 539d5d76..56a91593 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -79,6 +79,7 @@ TEST_F(CollectionTest, ExactPhraseSearch) {
std::vector<std::string> facets;
nlohmann::json results = collection->search("rocket launch", query_fields, "", facets, sort_fields, 0, 10);
ASSERT_EQ(5, results["hits"].size());
+ ASSERT_EQ(5, results["found"].get<uint32_t>());
/*
Sort by (match, diff, score)
@@ -101,6 +102,8 @@ TEST_F(CollectionTest, ExactPhraseSearch) {
// Check pagination
results = collection->search("rocket launch", query_fields, "", facets, sort_fields, 0, 3);
ASSERT_EQ(3, results["hits"].size());
+ ASSERT_EQ(4, results["found"].get<uint32_t>());
+
for(size_t i = 0; i < 3; i++) {
nlohmann::json result = results["hits"].at(i);
std::string id = ids.at(i);
@@ -264,6 +267,7 @@ TEST_F(CollectionTest, TextContainingAnActualTypo) {
std::vector<std::string> facets;
nlohmann::json results = collection->search("ISX what", query_fields, "", facets, sort_fields, 1, 4, FREQUENCY, false);
ASSERT_EQ(4, results["hits"].size());
+ ASSERT_EQ(4, results["found"].get<uint32_t>());
std::vector<std::string> ids = {"19", "6", "21", "8"};
@@ -277,6 +281,7 @@ TEST_F(CollectionTest, TextContainingAnActualTypo) {
// Record containing exact token match should appear first
results = collection->search("ISX", query_fields, "", facets, sort_fields, 1, 10, FREQUENCY, false);
ASSERT_EQ(8, results["hits"].size());
+ ASSERT_EQ(8, results["found"].get<uint32_t>());
ids = {"20", "19", "6", "3", "21", "4", "10", "8"};