Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Base work for supporting multiple indexable fields.
Kishore Nallan12276b651fd9
diff --git a/TODO.md b/TODO.mdindex 72ea0835..a133e411 100644--- a/TODO.md+++ b/TODO.md@@ -16,6 +16,8 @@- Prefix-search strings should not be null terminated- Make the search score computation customizable- string_utils::tokenize should not have max length+- only last token should be prefix searched+- art int search should support signed ints**API**diff --git a/include/art.h b/include/art.hindex 65ccb995..96742748 100644--- a/include/art.h+++ b/include/art.h@@ -242,9 +242,9 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,static int art_topk_iter(const art_node *root, token_ordering token_order, const int max_results,std::vector<art_leaf *> &results);-void encode_int(uint32_t n, unsigned char* chars);+void encode_int32(uint32_t n, unsigned char *chars);-int art_int_search(art_tree *t, uint32_t value, int compare, std::vector<const art_leaf*> & results);+int art_int32_search(art_tree *t, uint32_t value, int compare, std::vector<const art_leaf *> &results);#ifdef __cplusplus}diff --git a/include/collection.h b/include/collection.hindex 56a17aa8..9e49d485 100644--- a/include/collection.h+++ b/include/collection.h@@ -7,21 +7,30 @@#include <store.h>#include <topster.h>#include <json.hpp>+#include <field.h>class Collection {private:Store* store;+ std::string name;++ spp::sparse_hash_map<std::string, field> schema;+// Integer ID used internally for bitmaps - not exposed to the clientuint32_t seq_id;- art_tree t;+ spp::sparse_hash_map<std::string, art_tree*> index_map;+spp::sparse_hash_map<uint32_t, uint16_t> doc_scores;uint32_t next_seq_id();const std::string SEQ_ID_PREFIX = "SQ_";const std::string ID_PREFIX = "ID_";+ const std::string META_PREFIX = "MT_";++ const std::string FIELDS_KEY = META_PREFIX + "_fields";std::string get_seq_id_key(uint32_t seq_id);std::string get_id_key(std::string id);@@ -33,9 +42,13 @@ private:void search_candidates(std::vector<std::vector<art_leaf*>> & token_leaves, std::vector<nlohmann::json> & results,size_t & total_results, const size_t & max_results);+ void index_string_field(const std::string &field_name, art_tree *t, const nlohmann::json &document, uint32_t seq_id) const;++ void index_int32_field(const std::string &field_name, art_tree *t, const nlohmann::json &document, uint32_t seq_id) const;+public:Collection() = delete;- Collection(std::string state_dir_path);+ Collection(const std::string & state_dir_path, const std::string & name, const std::vector<field> & fields);~Collection();std::string add(std::string json_str);std::vector<nlohmann::json> search(std::string query, const int num_typos, const size_t num_results,diff --git a/include/field.h b/include/field.hnew file mode 100644index 00000000..3aaa2394--- /dev/null+++ b/include/field.h@@ -0,0 +1,15 @@+#include <string>++enum field_type {+ INT32,+ STRING+};++struct field {+ std::string name;+ field_type type;++ field(std::string name, field_type type): name(name), type(type) {++ }+};\ No newline at end of filediff --git a/src/art.cpp b/src/art.cppindex 2db74985..f0b932fd 100644--- a/src/art.cpp+++ b/src/art.cpp@@ -1349,7 +1349,7 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,return 0;}-void encode_int(uint32_t n, unsigned char* chars) {+void encode_int32(uint32_t n, unsigned char *chars) {unsigned char symbols[16] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};@@ -1551,9 +1551,9 @@ void art_int_fuzzy_recurse(art_node *n, int depth, unsigned char* int_str, int iart_int_fuzzy_children(n, depth, int_str, int_str_len, compare, results);}-int art_int_search(art_tree *t, uint32_t value, int compare, std::vector<const art_leaf*> & results) {+int art_int32_search(art_tree *t, uint32_t value, int compare, std::vector<const art_leaf *> &results) {unsigned char chars[9];- encode_int(value, chars);+ encode_int32(value, chars);art_int_fuzzy_recurse(t->root, 0, chars, 9, compare, results);return 0;}diff --git a/src/collection.cpp b/src/collection.cppindex 021daf10..05b1affa 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -11,14 +11,33 @@#include "art.h"#include "json.hpp"-Collection::Collection(std::string state_dir_path): seq_id(0) {+Collection::Collection(const std::string & state_dir_path, const std::string & name,+ const std::vector<field> & fields): seq_id(0), name(name) {store = new Store(state_dir_path);- art_tree_init(&t);++ nlohmann::json fields_json = nlohmann::json::array();++ for(const field& field: fields) {+ art_tree *t = new art_tree;+ art_tree_init(t);++ fields_json.push_back(field.name);+ index_map.emplace(field.name, t);+ schema.emplace(field.name, field);+ }++ store->insert(FIELDS_KEY, fields_json.dump());}Collection::~Collection() {delete store;- art_tree_destroy(&t);++ for(std::pair<std::string, field> name_field: schema) {+ art_tree *t = index_map.at(name_field.first);+ art_tree_destroy(t);+ }++ schema.clear();}uint32_t Collection::next_seq_id() {@@ -38,13 +57,54 @@ std::string Collection::add(std::string json_str) {store->insert(get_seq_id_key(seq_id), document.dump());store->insert(get_id_key(document["id"]), seq_id_str);+ for(const std::pair<std::string, field> & field_pair: schema) {+ const std::string & field_name = field_pair.first;+ art_tree *t = index_map.at(field_name);++ if(field_pair.second.type == STRING) {+ index_string_field(field_name, t, document, seq_id);+ } else if(field_pair.second.type == INT32) {+ index_int32_field(field_name, t, document, seq_id);+ }+ }++ doc_scores[seq_id] = document["points"];+ return document["id"];+}++void Collection::index_int32_field(const std::string &field_name, art_tree *t, const nlohmann::json &document,+ uint32_t seq_id) const {+ uint32_t value = document[field_name];+ unsigned char key[9];+ encode_int32(value, key);++ uint32_t num_hits = 0;+ art_leaf* leaf = (art_leaf *) art_search(t, key, 9);+ if(leaf != NULL) {+ num_hits = leaf->values->ids.getLength();+ }++ num_hits += 1;++ art_document art_doc;+ art_doc.id = seq_id;+ art_doc.score = document["points"];+ art_doc.offsets_len = 0;+ art_doc.offsets = nullptr;++ art_insert(t, key, 9, &art_doc, num_hits);+}++void Collection::index_string_field(const std::string &field_name, art_tree *t, const nlohmann::json &document,+ uint32_t seq_id) const {+ std::string text = document[field_name];std::vector<std::string> tokens;- StringUtils::tokenize(document["title"], tokens, " ", true);+ StringUtils::tokenize(text, tokens, " ", true);std::unordered_map<std::string, std::vector<uint32_t>> token_to_offsets;for(uint32_t i=0; i<tokens.size(); i++) {auto token = tokens[i];- std::transform(token.begin(), token.end(), token.begin(), ::tolower);+ transform(token.begin(), token.end(), token.begin(), tolower);token_to_offsets[token].push_back(i);}@@ -60,7 +120,7 @@ std::string Collection::add(std::string json_str) {const unsigned char *key = (const unsigned char *) kv.first.c_str();int key_len = (int) kv.first.length() + 1; // for the terminating \0 char- art_leaf* leaf = (art_leaf *) art_search(&t, key, key_len);+ art_leaf* leaf = (art_leaf *) art_search(t, key, key_len);if(leaf != NULL) {num_hits = leaf->values->ids.getLength();}@@ -71,13 +131,9 @@ std::string Collection::add(std::string json_str) {art_doc.offsets[i] = kv.second[i];}- art_insert(&t, key, key_len, &art_doc, num_hits);+ art_insert(t, key, key_len, &art_doc, num_hits);delete art_doc.offsets;}-- doc_scores[seq_id] = document["points"];-- return document["id"];}void Collection::search_candidates(std::vector<std::vector<art_leaf*>> & token_leaves,@@ -209,7 +265,7 @@ std::vector<nlohmann::json> Collection::search(std::string query, const int num_leaves = token_cache[token_cost_hash];} else {int token_len = prefix ? (int) token.length() : (int) token.length() + 1;- art_fuzzy_search(&t, (const unsigned char *) token.c_str(), token_len,+ art_fuzzy_search(index_map.at("title"), (const unsigned char *) token.c_str(), token_len,costs[token_index], 3, token_order, prefix, leaves);if(!leaves.empty()) {token_cache.emplace(token_cost_hash, leaves);@@ -422,7 +478,7 @@ void Collection::remove(std::string id) {const unsigned char *key = (const unsigned char *) token.c_str();int key_len = (int) (token.length() + 1);- art_leaf* leaf = (art_leaf *) art_search(&t, key, key_len);+ art_leaf* leaf = (art_leaf *) art_search(index_map.at("title"), key, key_len);if(leaf != NULL) {uint32_t seq_id_values[1] = {seq_id};@@ -453,7 +509,7 @@ void Collection::remove(std::string id) {std::cout << "----" << std::endl;*/if(leaf->values->ids.getLength() == 0) {- art_delete(&t, key, key_len);+ art_delete(index_map.at("title"), key, key_len);}}}@@ -468,4 +524,4 @@ std::string Collection::get_seq_id_key(uint32_t seq_id) {std::string Collection::get_id_key(std::string id) {return ID_PREFIX+id;-}+}\ No newline at end of filediff --git a/src/main/server.cpp b/src/main/server.cppindex c1b6b197..6cf3f872 100644--- a/src/main/server.cpp+++ b/src/main/server.cpp@@ -62,15 +62,27 @@ static int get_search(h2o_handler_t *self, h2o_req_t *req) {std::string query_str(query.base, query.len);std::map<std::string, std::string> query_map = parse_query(query_str);const char *NUM_TYPOS = "num_typos";+ const char *PREFIX = "prefix";+ const char *TOKEN_ORDERING = "token_ordering";if(query_map.count(NUM_TYPOS) == 0) {query_map[NUM_TYPOS] = "2";}+ if(query_map.count(PREFIX) == 0) {+ query_map[PREFIX] = "false";+ }++ if(query_map.count(TOKEN_ORDERING) == 0) {+ query_map[TOKEN_ORDERING] = "FREQUENCY";+ }++ token_ordering token_order = (query_map[TOKEN_ORDERING] == "MAX_SCORE") ? MAX_SCORE : FREQUENCY;+printf("Query: %s\n", query_map["q"].c_str());auto begin = std::chrono::high_resolution_clock::now();- std::vector<nlohmann::json> results = collection->search(query_map["q"], std::stoi(query_map[NUM_TYPOS]), 100, FREQUENCY, false);+ std::vector<nlohmann::json> results = collection->search(query_map["q"], std::stoi(query_map[NUM_TYPOS]), 100, token_order, false);nlohmann::json json_array = nlohmann::json::array();for(nlohmann::json& result: results) {json_array.push_back(result);diff --git a/test/art_test.cpp b/test/art_test.cppindex c8e3df3d..e1c0867d 100644--- a/test/art_test.cpp+++ b/test/art_test.cpp@@ -685,35 +685,35 @@ TEST(ArtTest, test_encode_int) {// 175 => 0000,0000,0000,0000,0000,0000,1010,1111,\0unsigned char chars_175[9] = {0, 0, 0, 0, 0, 0, 10, 15, 46};- encode_int(175, chars);+ encode_int32(175, chars);for(uint32_t i = 0; i < 9; i++) {ASSERT_EQ(chars_175[i], chars[i]);}// 0 => 0000,0000,0000,0000,0000,0000,0000,0000,\0unsigned char chars_0[9] = {0, 0, 0, 0, 0, 0, 0, 0, 46};- encode_int(0, chars);+ encode_int32(0, chars);for(uint32_t i = 0; i < 9; i++) {ASSERT_EQ(chars_0[i], chars[i]);}// 255 => 0000,0000,0000,0000,0000,0000,1111,1111,\0unsigned char chars_255[9] = {0, 0, 0, 0, 0, 0, 15, 15, 46};- encode_int(255, chars);+ encode_int32(255, chars);for(uint32_t i = 0; i < 9; i++) {ASSERT_EQ(chars_255[i], chars[i]);}// 4531 => 0000,0000,0000,0000,0001,0001,1011,0011,\0unsigned char chars_4531[9] = {0, 0, 0, 0, 1, 1, 11, 3, 46};- encode_int(4531, chars);+ encode_int32(4531, chars);for(uint32_t i = 0; i < 9; i++) {ASSERT_EQ(chars_4531[i], chars[i]);}// 1200000 => 0000,0000,0001,0010,0100,1111,1000,0000,\0unsigned char chars_1M[9] = {0, 0, 1, 2, 4, 15, 8, 0, 46};- encode_int(1200000, chars);+ encode_int32(1200000, chars);for(uint32_t i = 0; i < 9; i++) {ASSERT_EQ(chars_1M[i], chars[i]);}@@ -728,25 +728,25 @@ TEST(ArtTest, test_int_range_hundreds) {unsigned char chars[CHAR_LEN];for(uint32_t i = 100; i < 110; i++) {- encode_int(i, chars);+ encode_int32(i, chars);ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc, 1));}- encode_int(106, chars);+ encode_int32(106, chars);std::vector<const art_leaf*> results;- int res = art_int_search(&t, 106, 0, results);+ int res = art_int32_search(&t, 106, 0, results);ASSERT_TRUE(res == 0);ASSERT_EQ(1, results.size());results.clear();- res = art_int_search(&t, 106, 1, results);+ res = art_int32_search(&t, 106, 1, results);ASSERT_TRUE(res == 0);ASSERT_EQ(4, results.size());results.clear();- res = art_int_search(&t, 106, -1, results);+ res = art_int32_search(&t, 106, -1, results);ASSERT_TRUE(res == 0);ASSERT_EQ(7, results.size());@@ -764,11 +764,11 @@ TEST(ArtTest, test_int_range_millions) {unsigned char chars[CHAR_LEN];for(uint32_t i = 0; i < 1000000; i++) {- encode_int(i, chars);+ encode_int32(i, chars);ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc, 1));}- encode_int(5, chars);+ encode_int32(5, chars);/*std::cout << std::endl;for(uint32_t i = 0; i < CHAR_LEN; i++) {std::cout << (int)chars[i] << ", ";@@ -780,50 +780,50 @@ TEST(ArtTest, test_int_range_millions) {// ==for(uint32_t i = 0; i < 6; i++) {results.clear();- art_int_search(&t, (uint32_t) pow(10, i), 0, results);+ art_int32_search(&t, (uint32_t) pow(10, i), 0, results);ASSERT_EQ(1, results.size());results.clear();- art_int_search(&t, (uint32_t) (pow(10, i) + 7), 0, results);+ art_int32_search(&t, (uint32_t) (pow(10, i) + 7), 0, results);ASSERT_EQ(1, results.size());}results.clear();- art_int_search(&t, 1000000-1, 0, results);+ art_int32_search(&t, 1000000 - 1, 0, results);ASSERT_EQ(1, results.size());// >=results.clear();- art_int_search(&t, 1000000-5, 1, results);+ art_int32_search(&t, 1000000 - 5, 1, results);ASSERT_EQ(5, results.size());results.clear();- art_int_search(&t, 1000000-1, 1, results);+ art_int32_search(&t, 1000000 - 1, 1, results);ASSERT_EQ(1, results.size());results.clear();- art_int_search(&t, 1000000, 1, results);+ art_int32_search(&t, 1000000, 1, results);ASSERT_EQ(0, results.size());results.clear();- art_int_search(&t, 5, 1, results);+ art_int32_search(&t, 5, 1, results);ASSERT_EQ(1000000-5, results.size());// <=results.clear();- art_int_search(&t, 1000000-5, -1, results);+ art_int32_search(&t, 1000000 - 5, -1, results);ASSERT_EQ(1000000-5+1, results.size());results.clear();- art_int_search(&t, 1000000-1, -1, results);+ art_int32_search(&t, 1000000 - 1, -1, results);ASSERT_EQ(1000000, results.size());results.clear();- art_int_search(&t, 1000000, -1, results);+ art_int32_search(&t, 1000000, -1, results);ASSERT_EQ(1000000, results.size());results.clear();- art_int_search(&t, 5, -1, results);+ art_int32_search(&t, 5, -1, results);ASSERT_EQ(5+1, results.size());}@@ -837,15 +837,15 @@ TEST(ArtTest, test_int_range_byte_boundary) {unsigned char chars[CHAR_LEN];for(uint32_t i = 200; i < 300; i++) {- encode_int(i, chars);+ encode_int32(i, chars);ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)chars, CHAR_LEN, &doc, 1));}- encode_int(255, chars);+ encode_int32(255, chars);std::vector<const art_leaf*> results;results.clear();- art_int_search(&t, 255, 1, results);+ art_int32_search(&t, 255, 1, results);ASSERT_EQ(45, results.size());/*std::cout << std::endl;diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 818b96e7..fb7334e2 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -10,7 +10,8 @@ protected:virtual void SetUp() {std::ifstream infile("/Users/kishore/others/wreally/typesense/test/documents.jsonl");- collection = new Collection("/tmp/typesense_test/collection");+ std::vector<field> fields = {field("title", field_type::STRING)};+ collection = new Collection("/tmp/typesense_test/collection", "collection", fields);std::string json_line;