Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Base work for supporting multiple indexable fields.

Kishore Nallan12276b651fd9
diff --git a/TODO.md b/TODO.md
index 72ea0835..a133e411 100644
--- a/TODO.md
+++ b/TODO.md
@@ -16,6 +16,8 @@
- Prefix-search strings should not be null terminated
- Make the search score computation customizable
- string_utils::tokenize should not have max length
+- only last token should be prefix searched
+- art int search should support signed ints
**API**
diff --git a/include/art.h b/include/art.h
index 65ccb995..96742748 100644
--- a/include/art.h
+++ b/include/art.h
@@ -242,9 +242,9 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,
static int art_topk_iter(const art_node *root, token_ordering token_order, const int max_results,
std::vector<art_leaf *> &results);
-void encode_int(uint32_t n, unsigned char* chars);
+void encode_int32(uint32_t n, unsigned char *chars);
-int art_int_search(art_tree *t, uint32_t value, int compare, std::vector<const art_leaf*> & results);
+int art_int32_search(art_tree *t, uint32_t value, int compare, std::vector<const art_leaf *> &results);
#ifdef __cplusplus
}
diff --git a/include/collection.h b/include/collection.h
index 56a17aa8..9e49d485 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -7,21 +7,30 @@
#include <store.h>
#include <topster.h>
#include <json.hpp>
+#include <field.h>
class Collection {
private:
Store* store;
+ std::string name;
+
+ spp::sparse_hash_map<std::string, field> schema;
+
// Integer ID used internally for bitmaps - not exposed to the client
uint32_t seq_id;
- art_tree t;
+ spp::sparse_hash_map<std::string, art_tree*> index_map;
+
spp::sparse_hash_map<uint32_t, uint16_t> doc_scores;
uint32_t next_seq_id();
const std::string SEQ_ID_PREFIX = "SQ_";
const std::string ID_PREFIX = "ID_";
+ const std::string META_PREFIX = "MT_";
+
+ const std::string FIELDS_KEY = META_PREFIX + "_fields";
std::string get_seq_id_key(uint32_t seq_id);
std::string get_id_key(std::string id);
@@ -33,9 +42,13 @@ private:
void search_candidates(std::vector<std::vector<art_leaf*>> & token_leaves, std::vector<nlohmann::json> & results,
size_t & total_results, const size_t & max_results);
+ void index_string_field(const std::string &field_name, art_tree *t, const nlohmann::json &document, uint32_t seq_id) const;
+
+ void index_int32_field(const std::string &field_name, art_tree *t, const nlohmann::json &document, uint32_t seq_id) const;
+
public:
Collection() = delete;
- Collection(std::string state_dir_path);
+ Collection(const std::string & state_dir_path, const std::string & name, const std::vector<field> & fields);
~Collection();
std::string add(std::string json_str);
std::vector<nlohmann::json> search(std::string query, const int num_typos, const size_t num_results,
diff --git a/include/field.h b/include/field.h
new file mode 100644
index 00000000..3aaa2394
--- /dev/null
+++ b/include/field.h
@@ -0,0 +1,15 @@
+#include <string>
+
+enum field_type {
+ INT32,
+ STRING
+};
+
+struct field {
+ std::string name;
+ field_type type;
+
+ field(std::string name, field_type type): name(name), type(type) {
+
+ }
+};
\ No newline at end of file
diff --git a/src/art.cpp b/src/art.cpp
index 2db74985..f0b932fd 100644
--- a/src/art.cpp
+++ b/src/art.cpp
@@ -1349,7 +1349,7 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,
return 0;
}
-void encode_int(uint32_t n, unsigned char* chars) {
+void encode_int32(uint32_t n, unsigned char *chars) {
unsigned char symbols[16] = {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
};
@@ -1551,9 +1551,9 @@ void art_int_fuzzy_recurse(art_node *n, int depth, unsigned char* int_str, int i
art_int_fuzzy_children(n, depth, int_str, int_str_len, compare, results);
}
-int art_int_search(art_tree *t, uint32_t value, int compare, std::vector<const art_leaf*> & results) {
+int art_int32_search(art_tree *t, uint32_t value, int compare, std::vector<const art_leaf *> &results) {
unsigned char chars[9];
- encode_int(value, chars);
+ encode_int32(value, chars);
art_int_fuzzy_recurse(t->root, 0, chars, 9, compare, results);
return 0;
}
diff --git a/src/collection.cpp b/src/collection.cpp
index 021daf10..05b1affa 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -11,14 +11,33 @@
#include "art.h"
#include "json.hpp"
-Collection::Collection(std::string state_dir_path): seq_id(0) {
+Collection::Collection(const std::string & state_dir_path, const std::string & name,
+ const std::vector<field> & fields): seq_id(0), name(name) {
store = new Store(state_dir_path);
- art_tree_init(&t);
+
+ nlohmann::json fields_json = nlohmann::json::array();
+
+ for(const field& field: fields) {
+ art_tree *t = new art_tree;
+ art_tree_init(t);
+
+ fields_json.push_back(field.name);
+ index_map.emplace(field.name, t);
+ schema.emplace(field.name, field);
+ }
+
+ store->insert(FIELDS_KEY, fields_json.dump());
}
Collection::~Collection() {
delete store;
- art_tree_destroy(&t);
+
+ for(std::pair<std::string, field> name_field: schema) {
+ art_tree *t = index_map.at(name_field.first);
+ art_tree_destroy(t);
+ }
+
+ schema.clear();
}
uint32_t Collection::next_seq_id() {
@@ -38,13 +57,54 @@ std::string Collection::add(std::string json_str) {
store->insert(get_seq_id_key(seq_id), document.dump());
store->insert(get_id_key(document["id"]), seq_id_str);
+ for(const std::pair<std::string, field> & field_pair: schema) {
+ const std::string & field_name = field_pair.first;
+ art_tree *t = index_map.at(field_name);
+
+ if(field_pair.second.type == STRING) {
+ index_string_field(field_name, t, document, seq_id);
+ } else if(field_pair.second.type == INT32) {
+ index_int32_field(field_name, t, document, seq_id);
+ }
+ }
+
+ doc_scores[seq_id] = document["points"];
+ return document["id"];
+}
+
+void Collection::index_int32_field(const std::string &field_name, art_tree *t, const nlohmann::json &document,
+ uint32_t seq_id) const {
+ uint32_t value = document[field_name];
+ unsigned char key[9];
+ encode_int32(value, key);
+
+ uint32_t num_hits = 0;
+ art_leaf* leaf = (art_leaf *) art_search(t, key, 9);
+ if(leaf != NULL) {
+ num_hits = leaf->values->ids.getLength();
+ }
+
+ num_hits += 1;
+
+ art_document art_doc;
+ art_doc.id = seq_id;
+ art_doc.score = document["points"];
+ art_doc.offsets_len = 0;
+ art_doc.offsets = nullptr;
+
+ art_insert(t, key, 9, &art_doc, num_hits);
+}
+
+void Collection::index_string_field(const std::string &field_name, art_tree *t, const nlohmann::json &document,
+ uint32_t seq_id) const {
+ std::string text = document[field_name];
std::vector<std::string> tokens;
- StringUtils::tokenize(document["title"], tokens, " ", true);
+ StringUtils::tokenize(text, tokens, " ", true);
std::unordered_map<std::string, std::vector<uint32_t>> token_to_offsets;
for(uint32_t i=0; i<tokens.size(); i++) {
auto token = tokens[i];
- std::transform(token.begin(), token.end(), token.begin(), ::tolower);
+ transform(token.begin(), token.end(), token.begin(), tolower);
token_to_offsets[token].push_back(i);
}
@@ -60,7 +120,7 @@ std::string Collection::add(std::string json_str) {
const unsigned char *key = (const unsigned char *) kv.first.c_str();
int key_len = (int) kv.first.length() + 1; // for the terminating \0 char
- art_leaf* leaf = (art_leaf *) art_search(&t, key, key_len);
+ art_leaf* leaf = (art_leaf *) art_search(t, key, key_len);
if(leaf != NULL) {
num_hits = leaf->values->ids.getLength();
}
@@ -71,13 +131,9 @@ std::string Collection::add(std::string json_str) {
art_doc.offsets[i] = kv.second[i];
}
- art_insert(&t, key, key_len, &art_doc, num_hits);
+ art_insert(t, key, key_len, &art_doc, num_hits);
delete art_doc.offsets;
}
-
- doc_scores[seq_id] = document["points"];
-
- return document["id"];
}
void Collection::search_candidates(std::vector<std::vector<art_leaf*>> & token_leaves,
@@ -209,7 +265,7 @@ std::vector<nlohmann::json> Collection::search(std::string query, const int num_
leaves = token_cache[token_cost_hash];
} else {
int token_len = prefix ? (int) token.length() : (int) token.length() + 1;
- art_fuzzy_search(&t, (const unsigned char *) token.c_str(), token_len,
+ art_fuzzy_search(index_map.at("title"), (const unsigned char *) token.c_str(), token_len,
costs[token_index], 3, token_order, prefix, leaves);
if(!leaves.empty()) {
token_cache.emplace(token_cost_hash, leaves);
@@ -422,7 +478,7 @@ void Collection::remove(std::string id) {
const unsigned char *key = (const unsigned char *) token.c_str();
int key_len = (int) (token.length() + 1);
- art_leaf* leaf = (art_leaf *) art_search(&t, key, key_len);
+ art_leaf* leaf = (art_leaf *) art_search(index_map.at("title"), key, key_len);
if(leaf != NULL) {
uint32_t seq_id_values[1] = {seq_id};
@@ -453,7 +509,7 @@ void Collection::remove(std::string id) {
std::cout << "----" << std::endl;*/
if(leaf->values->ids.getLength() == 0) {
- art_delete(&t, key, key_len);
+ art_delete(index_map.at("title"), key, key_len);
}
}
}
@@ -468,4 +524,4 @@ std::string Collection::get_seq_id_key(uint32_t seq_id) {
std::string Collection::get_id_key(std::string id) {
return ID_PREFIX+id;
-}
+}
\ No newline at end of file
diff --git a/src/main/server.cpp b/src/main/server.cpp
index c1b6b197..6cf3f872 100644
--- a/src/main/server.cpp
+++ b/src/main/server.cpp
@@ -62,15 +62,27 @@ static int get_search(h2o_handler_t *self, h2o_req_t *req) {
std::string query_str(query.base, query.len);
std::map<std::string, std::string> query_map = parse_query(query_str);
const char *NUM_TYPOS = "num_typos";
+ const char *PREFIX = "prefix";
+ const char *TOKEN_ORDERING = "token_ordering";
if(query_map.count(NUM_TYPOS) == 0) {
query_map[NUM_TYPOS] = "2";
}
+ if(query_map.count(PREFIX) == 0) {
+ query_map[PREFIX] = "false";
+ }
+
+ if(query_map.count(TOKEN_ORDERING) == 0) {
+ query_map[TOKEN_ORDERING] = "FREQUENCY";
+ }
+
+ token_ordering token_order = (query_map[TOKEN_ORDERING] == "MAX_SCORE") ? MAX_SCORE : FREQUENCY;
+
printf("Query: %s\n", query_map["q"].c_str());
auto begin = std::chrono::high_resolution_clock::now();
- std::vector<nlohmann::json> results = collection->search(query_map["q"], std::stoi(query_map[NUM_TYPOS]), 100, FREQUENCY, false);
+ std::vector<nlohmann::json> results = collection->search(query_map["q"], std::stoi(query_map[NUM_TYPOS]), 100, token_order, false);
nlohmann::json json_array = nlohmann::json::array();
for(nlohmann::json& result: results) {
json_array.push_back(result);
diff --git a/test/art_test.cpp b/test/art_test.cpp
index c8e3df3d..e1c0867d 100644
--- a/test/art_test.cpp
+++ b/test/art_test.cpp
@@ -685,35 +685,35 @@ TEST(ArtTest, test_encode_int) {
// 175 => 0000,0000,0000,0000,0000,0000,1010,1111,\0
unsigned char chars_175[9] = {0, 0, 0, 0, 0, 0, 10, 15, 46};
- encode_int(175, chars);
+ encode_int32(175, chars);
for(uint32_t i = 0; i < 9; i++) {
ASSERT_EQ(chars_175[i], chars[i]);
}
// 0 => 0000,0000,0000,0000,0000,0000,0000,0000,\0
unsigned char chars_0[9] = {0, 0, 0, 0, 0, 0, 0, 0, 46};
- encode_int(0, chars);
+ encode_int32(0, chars);
for(uint32_t i = 0; i < 9; i++) {
ASSERT_EQ(chars_0[i], chars[i]);
}
// 255 => 0000,0000,0000,0000,0000,0000,1111,1111,\0
unsigned char chars_255[9] = {0, 0, 0, 0, 0, 0, 15, 15, 46};
- encode_int(255, chars);
+ encode_int32(255, chars);
for(uint32_t i = 0; i < 9; i++) {
ASSERT_EQ(chars_255[i], chars[i]);
}
// 4531 => 0000,0000,0000,0000,0001,0001,1011,0011,\0
unsigned char chars_4531[9] = {0, 0, 0, 0, 1, 1, 11, 3, 46};
- encode_int(4531, chars);
+ encode_int32(4531, chars);
for(uint32_t i = 0; i < 9; i++) {
ASSERT_EQ(chars_4531[i], chars[i]);
}
// 1200000 => 0000,0000,0001,0010,0100,1111,1000,0000,\0
unsigned char chars_1M[9] = {0, 0, 1, 2, 4, 15, 8, 0, 46};
- encode_int(1200000, chars);
+ encode_int32(1200000, chars);
for(uint32_t i = 0; i < 9; i++) {
ASSERT_EQ(chars_1M[i], chars[i]);
}
@@ -728,25 +728,25 @@ TEST(ArtTest, test_int_range_hundreds) {
unsigned char chars[CHAR_LEN];
for(uint32_t i = 100; i < 110; i++) {
- encode_int(i, chars);
+ encode_int32(i, chars);
ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc, 1));
}
- encode_int(106, chars);
+ encode_int32(106, chars);
std::vector<const art_leaf*> results;
- int res = art_int_search(&t, 106, 0, results);
+ int res = art_int32_search(&t, 106, 0, results);
ASSERT_TRUE(res == 0);
ASSERT_EQ(1, results.size());
results.clear();
- res = art_int_search(&t, 106, 1, results);
+ res = art_int32_search(&t, 106, 1, results);
ASSERT_TRUE(res == 0);
ASSERT_EQ(4, results.size());
results.clear();
- res = art_int_search(&t, 106, -1, results);
+ res = art_int32_search(&t, 106, -1, results);
ASSERT_TRUE(res == 0);
ASSERT_EQ(7, results.size());
@@ -764,11 +764,11 @@ TEST(ArtTest, test_int_range_millions) {
unsigned char chars[CHAR_LEN];
for(uint32_t i = 0; i < 1000000; i++) {
- encode_int(i, chars);
+ encode_int32(i, chars);
ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc, 1));
}
- encode_int(5, chars);
+ encode_int32(5, chars);
/*std::cout << std::endl;
for(uint32_t i = 0; i < CHAR_LEN; i++) {
std::cout << (int)chars[i] << ", ";
@@ -780,50 +780,50 @@ TEST(ArtTest, test_int_range_millions) {
// ==
for(uint32_t i = 0; i < 6; i++) {
results.clear();
- art_int_search(&t, (uint32_t) pow(10, i), 0, results);
+ art_int32_search(&t, (uint32_t) pow(10, i), 0, results);
ASSERT_EQ(1, results.size());
results.clear();
- art_int_search(&t, (uint32_t) (pow(10, i) + 7), 0, results);
+ art_int32_search(&t, (uint32_t) (pow(10, i) + 7), 0, results);
ASSERT_EQ(1, results.size());
}
results.clear();
- art_int_search(&t, 1000000-1, 0, results);
+ art_int32_search(&t, 1000000 - 1, 0, results);
ASSERT_EQ(1, results.size());
// >=
results.clear();
- art_int_search(&t, 1000000-5, 1, results);
+ art_int32_search(&t, 1000000 - 5, 1, results);
ASSERT_EQ(5, results.size());
results.clear();
- art_int_search(&t, 1000000-1, 1, results);
+ art_int32_search(&t, 1000000 - 1, 1, results);
ASSERT_EQ(1, results.size());
results.clear();
- art_int_search(&t, 1000000, 1, results);
+ art_int32_search(&t, 1000000, 1, results);
ASSERT_EQ(0, results.size());
results.clear();
- art_int_search(&t, 5, 1, results);
+ art_int32_search(&t, 5, 1, results);
ASSERT_EQ(1000000-5, results.size());
// <=
results.clear();
- art_int_search(&t, 1000000-5, -1, results);
+ art_int32_search(&t, 1000000 - 5, -1, results);
ASSERT_EQ(1000000-5+1, results.size());
results.clear();
- art_int_search(&t, 1000000-1, -1, results);
+ art_int32_search(&t, 1000000 - 1, -1, results);
ASSERT_EQ(1000000, results.size());
results.clear();
- art_int_search(&t, 1000000, -1, results);
+ art_int32_search(&t, 1000000, -1, results);
ASSERT_EQ(1000000, results.size());
results.clear();
- art_int_search(&t, 5, -1, results);
+ art_int32_search(&t, 5, -1, results);
ASSERT_EQ(5+1, results.size());
}
@@ -837,15 +837,15 @@ TEST(ArtTest, test_int_range_byte_boundary) {
unsigned char chars[CHAR_LEN];
for(uint32_t i = 200; i < 300; i++) {
- encode_int(i, chars);
+ encode_int32(i, chars);
ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc, 1));
}
- encode_int(255, chars);
+ encode_int32(255, chars);
std::vector<const art_leaf*> results;
results.clear();
- art_int_search(&t, 255, 1, results);
+ art_int32_search(&t, 255, 1, results);
ASSERT_EQ(45, results.size());
/*std::cout << std::endl;
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 818b96e7..fb7334e2 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -10,7 +10,8 @@ protected:
virtual void SetUp() {
std::ifstream infile("/Users/kishore/others/wreally/typesense/test/documents.jsonl");
- collection = new Collection("/tmp/typesense_test/collection");
+ std::vector<field> fields = {field("title", field_type::STRING)};
+ collection = new Collection("/tmp/typesense_test/collection", "collection", fields);
std::string json_line;