Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Allow use of custom primary and secondary attributes for ranking.
Kishore Nallan8aaa9b174ffa
diff --git a/TODO.md b/TODO.mdindex a133e411..dfb7fdc9 100644--- a/TODO.md+++ b/TODO.md@@ -14,10 +14,13 @@- ~~Delete should remove from RocksDB~~- ~~Speed up UUID generation~~- Prefix-search strings should not be null terminated-- Make the search score computation customizable+- ~~Make the search score computation customizable~~- string_utils::tokenize should not have max length- only last token should be prefix searched- art int search should support signed ints+- storage key prefix should include collection name+- storage key prefix should include collection name+- use art for indexing score as well**API**diff --git a/include/topster.h b/include/topster.hindex cd02bed2..8257f390 100644--- a/include/topster.h+++ b/include/topster.h@@ -12,7 +12,9 @@ template <size_t MAX_SIZE=100>struct Topster {struct KV {uint64_t key;- uint64_t value;+ uint64_t match_score;+ int64_t primary_attr;+ int64_t secondary_attr;} data[MAX_SIZE];uint32_t size;@@ -27,25 +29,31 @@ struct Topster {b = c;}- void add(const uint64_t &key, const uint64_t &value){+ void add(const uint64_t &key, const uint64_t &match_score, const int64_t &primary_attr, const int64_t &secondary_attr){if (size >= MAX_SIZE) {- if(value <= data[0].value) {+ if(match_score <= data[0].match_score && primary_attr <= data[0].primary_attr &&+ secondary_attr <= data[0].secondary_attr) {// when incoming value is less than the smallest in the heap, ignorereturn;}data[0].key = key;- data[0].value = value;+ data[0].match_score = match_score;+ data[0].primary_attr = primary_attr;+ data[0].secondary_attr = secondary_attr;uint32_t i = 0;// sift to maintain heap propertywhile ((2*i+1) < MAX_SIZE) {uint32_t next = (uint32_t) (2 * i + 1);- if (next+1 < MAX_SIZE && data[next].value > data[next+1].value) {+ if (next+1 < MAX_SIZE && data[next].match_score > data[next+1].match_score &&+ data[next].primary_attr > data[next+1].primary_attr &&+ data[next].secondary_attr > data[next+1].secondary_attr) {next++;}- if (data[i].value > data[next].value) {+ if (data[i].match_score > data[next].match_score && data[i].primary_attr > data[next].primary_attr &&+ data[i].secondary_attr > data[next].secondary_attr) {swapMe(data[i], data[next]);} else {break;@@ -55,12 +63,15 @@ struct Topster {}} else {data[size].key = key;- data[size].value = value;+ data[size].match_score = match_score;+ data[size].primary_attr = primary_attr;+ data[size].secondary_attr = secondary_attr;size++;for (uint32_t i = size - 1; i > 0;) {uint32_t parent = (i-1)/2;- if (data[parent].value > data[i].value) {+ if (data[parent].match_score > data[i].match_score && data[parent].primary_attr > data[i].primary_attr &&+ data[parent].secondary_attr > data[i].secondary_attr) {swapMe(data[i], data[parent]);i = parent;} else {@@ -71,7 +82,10 @@ struct Topster {}static bool compare_values(const struct KV& i, const struct KV& j) {- return (i.value == j.value) ? i.key > j.key : i.value > j.value;+ if(i.match_score != j.match_score) return i.match_score > j.match_score;+ if(i.primary_attr != j.primary_attr) return i.primary_attr > j.primary_attr;+ if(i.secondary_attr != j.secondary_attr) return i.secondary_attr > j.secondary_attr;+ return i.key > j.key;}void sort() {diff --git a/src/collection.cpp b/src/collection.cppindex 05b1affa..42245767 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -390,9 +390,7 @@ void Collection::score_results(Topster<100> &topster, const std::vector<art_leafmscore = MatchScore::match_score(doc_id, token_positions);}- const uint64_t final_score = ((uint64_t)(mscore.words_present * 32 + (MAX_SEARCH_TOKENS - mscore.distance)) * UINT32_MAX) +- doc_scores.at(doc_id);-+ const uint64_t match_score = (uint64_t)(mscore.words_present * 32 + (MAX_SEARCH_TOKENS - mscore.distance));/*std::cout << "final_score: " << final_score << ", doc_id: " << doc_id << std::endl;uint32_t doc_score = doc_scores.at(doc_id);@@ -402,7 +400,7 @@ void Collection::score_results(Topster<100> &topster, const std::vector<art_leaf<< final_score << std::endl;*/- topster.add(doc_id, final_score);+ topster.add(doc_id, match_score, doc_scores.at(doc_id), 0);}}diff --git a/src/main/main.cpp b/src/main/main.cppindex aae50e08..b81baed2 100644--- a/src/main/main.cpp+++ b/src/main/main.cpp@@ -12,7 +12,8 @@using namespace std;int main() {- Collection *collection = new Collection("/tmp/typesense-data");+ std::vector<field> fields = {field("title", field_type::STRING)};+ Collection *collection = new Collection("/tmp/typesense-data", "collection", fields);std::ifstream infile("/Users/kishore/others/wreally/typesense/test/documents.jsonl");//std::ifstream infile("/Users/kishore/Downloads/hnstories.jsonl");diff --git a/src/main/server.cpp b/src/main/server.cppindex 6cf3f872..e2cf1483 100644--- a/src/main/server.cpp+++ b/src/main/server.cpp@@ -26,7 +26,8 @@static h2o_globalconf_t config;static h2o_context_t ctx;static h2o_accept_ctx_t accept_ctx;-static Collection *collection = new Collection("/tmp/typesense-data");+std::vector<field> fields = {field("title", field_type::STRING)};+static Collection *collection = new Collection("/tmp/typesense-data", "collection", fields);static h2o_pathconf_t *register_handler(h2o_hostconf_t *hostconf, const char *path,int (*on_req)(h2o_handler_t *, h2o_req_t *)) {