Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Allow use of custom primary and secondary attributes for ranking.

Kishore Nallan8aaa9b174ffa
diff --git a/TODO.md b/TODO.md
index a133e411..dfb7fdc9 100644
--- a/TODO.md
+++ b/TODO.md
@@ -14,10 +14,13 @@
- ~~Delete should remove from RocksDB~~
- ~~Speed up UUID generation~~
- Prefix-search strings should not be null terminated
-- Make the search score computation customizable
+- ~~Make the search score computation customizable~~
- string_utils::tokenize should not have max length
- only last token should be prefix searched
- art int search should support signed ints
+- storage key prefix should include collection name
+- storage key prefix should include collection name
+- use art for indexing score as well
**API**
diff --git a/include/topster.h b/include/topster.h
index cd02bed2..8257f390 100644
--- a/include/topster.h
+++ b/include/topster.h
@@ -12,7 +12,9 @@ template <size_t MAX_SIZE=100>
struct Topster {
struct KV {
uint64_t key;
- uint64_t value;
+ uint64_t match_score;
+ int64_t primary_attr;
+ int64_t secondary_attr;
} data[MAX_SIZE];
uint32_t size;
@@ -27,25 +29,31 @@ struct Topster {
b = c;
}
- void add(const uint64_t &key, const uint64_t &value){
+ void add(const uint64_t &key, const uint64_t &match_score, const int64_t &primary_attr, const int64_t &secondary_attr){
if (size >= MAX_SIZE) {
- if(value <= data[0].value) {
+ if(match_score <= data[0].match_score && primary_attr <= data[0].primary_attr &&
+ secondary_attr <= data[0].secondary_attr) {
// when incoming value is less than the smallest in the heap, ignore
return;
}
data[0].key = key;
- data[0].value = value;
+ data[0].match_score = match_score;
+ data[0].primary_attr = primary_attr;
+ data[0].secondary_attr = secondary_attr;
uint32_t i = 0;
// sift to maintain heap property
while ((2*i+1) < MAX_SIZE) {
uint32_t next = (uint32_t) (2 * i + 1);
- if (next+1 < MAX_SIZE && data[next].value > data[next+1].value) {
+ if (next+1 < MAX_SIZE && data[next].match_score > data[next+1].match_score &&
+ data[next].primary_attr > data[next+1].primary_attr &&
+ data[next].secondary_attr > data[next+1].secondary_attr) {
next++;
}
- if (data[i].value > data[next].value) {
+ if (data[i].match_score > data[next].match_score && data[i].primary_attr > data[next].primary_attr &&
+ data[i].secondary_attr > data[next].secondary_attr) {
swapMe(data[i], data[next]);
} else {
break;
@@ -55,12 +63,15 @@ struct Topster {
}
} else {
data[size].key = key;
- data[size].value = value;
+ data[size].match_score = match_score;
+ data[size].primary_attr = primary_attr;
+ data[size].secondary_attr = secondary_attr;
size++;
for (uint32_t i = size - 1; i > 0;) {
uint32_t parent = (i-1)/2;
- if (data[parent].value > data[i].value) {
+ if (data[parent].match_score > data[i].match_score && data[parent].primary_attr > data[i].primary_attr &&
+ data[parent].secondary_attr > data[i].secondary_attr) {
swapMe(data[i], data[parent]);
i = parent;
} else {
@@ -71,7 +82,10 @@ struct Topster {
}
static bool compare_values(const struct KV& i, const struct KV& j) {
- return (i.value == j.value) ? i.key > j.key : i.value > j.value;
+ if(i.match_score != j.match_score) return i.match_score > j.match_score;
+ if(i.primary_attr != j.primary_attr) return i.primary_attr > j.primary_attr;
+ if(i.secondary_attr != j.secondary_attr) return i.secondary_attr > j.secondary_attr;
+ return i.key > j.key;
}
void sort() {
diff --git a/src/collection.cpp b/src/collection.cpp
index 05b1affa..42245767 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -390,9 +390,7 @@ void Collection::score_results(Topster<100> &topster, const std::vector<art_leaf
mscore = MatchScore::match_score(doc_id, token_positions);
}
- const uint64_t final_score = ((uint64_t)(mscore.words_present * 32 + (MAX_SEARCH_TOKENS - mscore.distance)) * UINT32_MAX) +
- doc_scores.at(doc_id);
-
+ const uint64_t match_score = (uint64_t)(mscore.words_present * 32 + (MAX_SEARCH_TOKENS - mscore.distance));
/*
std::cout << "final_score: " << final_score << ", doc_id: " << doc_id << std::endl;
uint32_t doc_score = doc_scores.at(doc_id);
@@ -402,7 +400,7 @@ void Collection::score_results(Topster<100> &topster, const std::vector<art_leaf
<< final_score << std::endl;
*/
- topster.add(doc_id, final_score);
+ topster.add(doc_id, match_score, doc_scores.at(doc_id), 0);
}
}
diff --git a/src/main/main.cpp b/src/main/main.cpp
index aae50e08..b81baed2 100644
--- a/src/main/main.cpp
+++ b/src/main/main.cpp
@@ -12,7 +12,8 @@
using namespace std;
int main() {
- Collection *collection = new Collection("/tmp/typesense-data");
+ std::vector<field> fields = {field("title", field_type::STRING)};
+ Collection *collection = new Collection("/tmp/typesense-data", "collection", fields);
std::ifstream infile("/Users/kishore/others/wreally/typesense/test/documents.jsonl");
//std::ifstream infile("/Users/kishore/Downloads/hnstories.jsonl");
diff --git a/src/main/server.cpp b/src/main/server.cpp
index 6cf3f872..e2cf1483 100644
--- a/src/main/server.cpp
+++ b/src/main/server.cpp
@@ -26,7 +26,8 @@
static h2o_globalconf_t config;
static h2o_context_t ctx;
static h2o_accept_ctx_t accept_ctx;
-static Collection *collection = new Collection("/tmp/typesense-data");
+std::vector<field> fields = {field("title", field_type::STRING)};
+static Collection *collection = new Collection("/tmp/typesense-data", "collection", fields);
static h2o_pathconf_t *register_handler(h2o_hostconf_t *hostconf, const char *path,
int (*on_req)(h2o_handler_t *, h2o_req_t *)) {