Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Index and search on multi-valued string field.
Kishore Nallan385e8cb7a25a
diff --git a/TODO.md b/TODO.mdindex 126266ac..83126bd6 100644--- a/TODO.md+++ b/TODO.md@@ -20,8 +20,10 @@- ~~Multi field search tests~~- ~~storage key prefix should include collection name~~- Index and search on multi-valued field+- Proper field for ranking tokens+- Support nested fields via "."- ~~Restore records as well on restart (like for meta)~~-- drop collection should remove all records from the store+- ~~drop collection should remove all records from the store~~- Pagination parameter- UTF-8 support for fuzzy search- Facetsdiff --git a/include/collection.h b/include/collection.hindex e6d2d3ba..4047d093 100644--- a/include/collection.h+++ b/include/collection.h@@ -51,7 +51,9 @@ private:void search_candidates(int & token_rank, std::vector<std::vector<art_leaf*>> & token_leaves, Topster<100> & topster,size_t & total_results, size_t & num_found, const size_t & max_results);- void index_string_field(const std::string &field_name, art_tree *t, const nlohmann::json &document, uint32_t seq_id) const;+ void index_string_field(const std::string & text, uint32_t score, art_tree *t, uint32_t seq_id) const;++ void index_string_array_field(const std::string &field_name, art_tree *t, const nlohmann::json &document, uint32_t seq_id) const;void index_numeric_field(const std::string &field_name, const std::string & field_type,art_tree *t, const nlohmann::json &document, uint32_t seq_id) const;diff --git a/include/field.h b/include/field.hindex f2701eb6..add92964 100644--- a/include/field.h+++ b/include/field.h@@ -6,6 +6,7 @@ namespace field_types {static const std::string STRING = "STRING";static const std::string INT32 = "INT32";static const std::string INT64 = "INT64";+ static const std::string STRING_ARR = "STRING_ARR";}namespace fields {diff --git a/src/collection.cpp b/src/collection.cppindex 2499fe46..f5578006 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -54,9 +54,13 @@ void Collection::index_in_memory(const nlohmann::json &document, uint32_t seq_idart_tree *t = index_map.at(field_name);if(field_pair.second.type == field_types::STRING) {- index_string_field(field_name, t, document, seq_id);+ const std::string & text = document[field_name];+ uint32_t points = document["points"];+ index_string_field(text, points, t, seq_id);} else if(field_pair.second.type == field_types::INT32 || field_pair.second.type == field_types::INT64) {index_numeric_field(field_name, field_pair.second.type, t, document, seq_id);+ } else if(field_pair.second.type == field_types::STRING_ARR) {+ index_string_array_field(field_name, t, document, seq_id);}}if(rank_fields.size() > 0 && document.count(rank_fields[0])) {@@ -97,9 +101,7 @@ void Collection::index_numeric_field(const std::string &field_name, const std::sart_insert(t, key, KEY_LEN, &art_doc, num_hits);}-void Collection::index_string_field(const std::string &field_name, art_tree *t, const nlohmann::json &document,- uint32_t seq_id) const {- std::string text = document[field_name];+void Collection::index_string_field(const std::string & text, uint32_t score, art_tree *t, uint32_t seq_id) const {std::vector<std::string> tokens;StringUtils::tokenize(text, tokens, " ", true);@@ -113,7 +115,7 @@ void Collection::index_string_field(const std::string &field_name, art_tree *t,for(auto & kv: token_to_offsets) {art_document art_doc;art_doc.id = seq_id;- art_doc.score = document["points"];+ art_doc.score = score;art_doc.offsets_len = (uint32_t) kv.second.size();art_doc.offsets = new uint32_t[kv.second.size()];@@ -138,6 +140,15 @@ void Collection::index_string_field(const std::string &field_name, art_tree *t,}}+void Collection::index_string_array_field(const std::string &field_name, art_tree *t, const nlohmann::json &document,+ uint32_t seq_id) const {+ uint32_t points = document["points"];+ std::vector<std::string> strings = document[field_name];+ for(const std::string & str: strings) {+ index_string_field(str, points, t, seq_id);+ }+}+void Collection::search_candidates(int & token_rank, std::vector<std::vector<art_leaf*>> & token_leaves,Topster<100> & topster, size_t & total_results, size_t & num_found, const size_t & max_results) {const size_t combination_limit = 10;diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 04b0f7a0..d5fbdf76 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -61,7 +61,7 @@ TEST_F(CollectionTest, ExactSearchShouldBeStable) {// For two documents of the same score, the larger doc_id appears firststd::vector<std::string> ids = {"1", "6", "foo", "13", "10", "8", "16"};- for(size_t i = 0; i < results.size(); i++) {+ for(size_t i = 0; i < results["hits"].size(); i++) {nlohmann::json result = results["hits"].at(i);std::string id = ids.at(i);std::string result_id = result["id"];@@ -84,7 +84,7 @@ TEST_F(CollectionTest, ExactPhraseSearch) {std::vector<std::string> ids = {"8", "1", "17", "16", "13"};- for(size_t i = 0; i < results.size(); i++) {+ for(size_t i = 0; i < results["hits"].size(); i++) {nlohmann::json result = results["hits"].at(i);std::string id = ids.at(i);std::string result_id = result["id"];@@ -109,7 +109,7 @@ TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {std::vector<std::string> ids = {"2", "17"};- for(size_t i = 0; i < results.size(); i++) {+ for(size_t i = 0; i < results["hits"].size(); i++) {nlohmann::json result = results["hits"].at(i);std::string id = ids.at(i);std::string result_id = result["id"];@@ -120,7 +120,7 @@ TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {results = collection->search("DoesNotExist from", search_fields, 1, 10);ASSERT_EQ(2, results["hits"].size());- for(size_t i = 0; i < results.size(); i++) {+ for(size_t i = 0; i < results["hits"].size(); i++) {nlohmann::json result = results["hits"].at(i);std::string id = ids.at(i);std::string result_id = result["id"];@@ -132,7 +132,7 @@ TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {ASSERT_EQ(2, results["hits"].size());ids = {"2", "17"};- for(size_t i = 0; i < results.size(); i++) {+ for(size_t i = 0; i < results["hits"].size(); i++) {nlohmann::json result = results["hits"].at(i);std::string id = ids.at(i);std::string result_id = result["id"];@@ -154,7 +154,7 @@ TEST_F(CollectionTest, PartialPhraseSearch) {std::vector<std::string> ids = {"1", "8", "16", "17"};- for(size_t i = 0; i < results.size(); i++) {+ for(size_t i = 0; i < results["hits"].size(); i++) {nlohmann::json result = results["hits"].at(i);std::string result_id = result["id"];std::string id = ids.at(i);@@ -168,7 +168,7 @@ TEST_F(CollectionTest, QueryWithTypo) {std::vector<std::string> ids = {"19", "20", "21"};- for(size_t i = 0; i < results.size(); i++) {+ for(size_t i = 0; i < results["hits"].size(); i++) {nlohmann::json result = results["hits"].at(i);std::string result_id = result["id"];std::string id = ids.at(i);@@ -181,7 +181,7 @@ TEST_F(CollectionTest, QueryWithTypo) {ASSERT_EQ(3, results["hits"].size());- for(size_t i = 0; i < results.size(); i++) {+ for(size_t i = 0; i < results["hits"].size(); i++) {nlohmann::json result = results["hits"].at(i);std::string result_id = result["id"];std::string id = ids.at(i);@@ -194,7 +194,7 @@ TEST_F(CollectionTest, TypoTokenRankedByScoreAndFrequency) {ASSERT_EQ(2, results["hits"].size());std::vector<std::string> ids = {"22", "23"};- for(size_t i = 0; i < results.size(); i++) {+ for(size_t i = 0; i < results["hits"].size(); i++) {nlohmann::json result = results["hits"].at(i);std::string result_id = result["id"];std::string id = ids.at(i);@@ -205,7 +205,7 @@ TEST_F(CollectionTest, TypoTokenRankedByScoreAndFrequency) {ASSERT_EQ(3, results["hits"].size());ids = {"3", "12", "24"};- for(size_t i = 0; i < results.size(); i++) {+ for(size_t i = 0; i < results["hits"].size(); i++) {nlohmann::json result = results["hits"].at(i);std::string result_id = result["id"];std::string id = ids.at(i);@@ -229,7 +229,7 @@ TEST_F(CollectionTest, TypoTokenRankedByScoreAndFrequency) {ASSERT_EQ(5, results["hits"].size());ids = {"3", "12", "24", "22", "23"};- for(size_t i = 0; i < results.size(); i++) {+ for(size_t i = 0; i < results["hits"].size(); i++) {nlohmann::json result = results["hits"].at(i);std::string result_id = result["id"];std::string id = ids.at(i);@@ -240,7 +240,7 @@ TEST_F(CollectionTest, TypoTokenRankedByScoreAndFrequency) {ASSERT_EQ(5, results["hits"].size());ids = {"22", "23", "3", "12", "24"};- for(size_t i = 0; i < results.size(); i++) {+ for(size_t i = 0; i < results["hits"].size(); i++) {nlohmann::json result = results["hits"].at(i);std::string result_id = result["id"];std::string id = ids.at(i);@@ -255,7 +255,7 @@ TEST_F(CollectionTest, TextContainingAnActualTypo) {std::vector<std::string> ids = {"19", "6", "21", "8"};- for(size_t i = 0; i < results.size(); i++) {+ for(size_t i = 0; i < results["hits"].size(); i++) {nlohmann::json result = results["hits"].at(i);std::string result_id = result["id"];std::string id = ids.at(i);@@ -268,7 +268,7 @@ TEST_F(CollectionTest, TextContainingAnActualTypo) {ids = {"20", "19", "6", "3", "21", "4", "10", "8"};- for(size_t i = 0; i < results.size(); i++) {+ for(size_t i = 0; i < results["hits"].size(); i++) {nlohmann::json result = results["hits"].at(i);std::string result_id = result["id"];std::string id = ids.at(i);@@ -281,7 +281,7 @@ TEST_F(CollectionTest, PrefixSearching) {ASSERT_EQ(2, results["hits"].size());std::vector<std::string> ids = {"12", "6"};- for(size_t i = 0; i < results.size(); i++) {+ for(size_t i = 0; i < results["hits"].size(); i++) {nlohmann::json result = results["hits"].at(i);std::string result_id = result["id"];std::string id = ids.at(i);@@ -292,7 +292,7 @@ TEST_F(CollectionTest, PrefixSearching) {ASSERT_EQ(2, results["hits"].size());ids = {"6", "12"};- for(size_t i = 0; i < results.size(); i++) {+ for(size_t i = 0; i < results["hits"].size(); i++) {nlohmann::json result = results["hits"].at(i);std::string result_id = result["id"];std::string id = ids.at(i);@@ -304,7 +304,8 @@ TEST_F(CollectionTest, MultipleFields) {Collection *coll_mul_fields;std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");- std::vector<field> fields = {field("title", field_types::STRING), field("starring", field_types::STRING)};+ std::vector<field> fields = {field("title", field_types::STRING), field("starring", field_types::STRING),+ field("cast", field_types::STRING_ARR)};std::vector<std::string> rank_fields = {"points"};coll_mul_fields = collectionManager.get_collection("coll_mul_fields");@@ -326,7 +327,7 @@ TEST_F(CollectionTest, MultipleFields) {std::vector<std::string> ids = {"3", "2", "1", "0"};- for(size_t i = 0; i < results.size(); i++) {+ for(size_t i = 0; i < results["hits"].size(); i++) {nlohmann::json result = results["hits"].at(i);std::string result_id = result["id"];std::string id = ids.at(i);@@ -341,7 +342,35 @@ TEST_F(CollectionTest, MultipleFields) {ids = {"15", "14", "12", "13"};- for(size_t i = 0; i < results.size(); i++) {+ for(size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["id"];+ std::string id = ids.at(i);+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }++ search_fields = {"starring", "title", "cast"};+ results = coll_mul_fields->search("ben affleck", search_fields, 0, 10, FREQUENCY, false);+ ASSERT_EQ(1, results["hits"].size());++ search_fields = {"cast"};+ results = coll_mul_fields->search("chris", search_fields, 0, 10, FREQUENCY, false);+ ASSERT_EQ(3, results["hits"].size());++ ids = {"6", "1", "7"};+ for(size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["id"];+ std::string id = ids.at(i);+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }++ search_fields = {"cast"};+ results = coll_mul_fields->search("chris pine", search_fields, 0, 10, FREQUENCY, false);+ ASSERT_EQ(3, results["hits"].size());++ ids = {"7", "6", "1"};+ for(size_t i = 0; i < results["hits"].size(); i++) {nlohmann::json result = results["hits"].at(i);std::string result_id = result["id"];std::string id = ids.at(i);diff --git a/test/multi_field_documents.jsonl b/test/multi_field_documents.jsonlindex 9782e632..71a7bda8 100644--- a/test/multi_field_documents.jsonl+++ b/test/multi_field_documents.jsonl@@ -1,18 +1,18 @@-{"title": "Wake Up, Ron Burgundy: The Lost Movie", "starring": "Will Ferrell", "points": 62 }-{"title": "Anchorman 2: The Legend Continues", "starring": "Will Ferrell", "points": 63 }-{"title": "There Will Be Blood", "starring": "Daniel Day-Lewis", "points": 81 }-{"title": "Good Will Hunting", "starring": "Robin Williams", "points": 83 }-{"title": "The Adventures of Huck Finn", "starring": "Ron Perlman", "points": 58 }-{"title": "Percy Jackson: Sea of Monsters", "starring": "Ron Perlman", "points": 59 }-{"title": "Captain America: The Winter Soldier", "starring": "Samuel L. Jackson", "points": 78 }-{"title": "Quantum Quest: A Cassini Space Odyssey", "starring": "Samuel L. Jackson", "points": 52 }-{"title": "Scott Pilgrim vs. the World", "starring": "Michael Cera", "points": 75 }-{"title": "Homeland Security", "starring": "Scott Glenn", "points": 43 }-{"title": "The Paperboy", "starring": "Scott Glenn", "points": 58 }-{"title": "The Silence of the Lambs", "starring": "Scott Glenn", "points": 86 }-{"title": "Confessions of a Shopaholic", "starring": "Kristin Scott Thomas", "points": 59 }-{"title": "The Woman in the Fifth", "starring": "Kristin Scott Thomas", "points": 53 }-{"title": "Odd Thomas", "starring": "Matthew Page", "points": 69 }-{"title": "Suffering Man's Charity", "starring": "Henry Thomas", "points": 69 }-{"title": "The Gospel According to St. Matthew", "starring": "Paola Tedesco", "points": 79 }-{"title": "Halloween 5: The Revenge of Michael Myers", "starring": "Donald Pleasence", "points": 52 }\ No newline at end of file+{"title": "Wake Up, Ron Burgundy: The Lost Movie", "starring": "Will Ferrell", "points": 62, "cast": [ "Adam McKay", "Steve Carell", "Paul Rudd"] }+{"title": "Anchorman 2: The Legend Continues", "starring": "Will Ferrell", "points": 63, "cast": ["Josh Lawson", "Chris Parnell"] }+{"title": "There Will Be Blood", "starring": "Daniel Day-Lewis", "points": 81, "cast": ["Martin Stringer", "Jacob Stringer", "Harrison Taylor"] }+{"title": "Good Will Hunting", "starring": "Robin Williams", "points": 83, "cast": ["Matt Damon", "Ben Affleck", "Minnie Driver"] }+{"title": "The Adventures of Huck Finn", "starring": "Ron Perlman", "points": 58, "cast": ["Elijah Wood", "Courtney B. Vance", "Robbie Coltrane"] }+{"title": "Percy Jackson: Sea of Monsters", "starring": "Ron Perlman", "points": 59, "cast": ["Logan Lerman", "Alexandra Daddario"] }+{"title": "Captain America: The Winter Soldier", "starring": "Samuel L. Jackson", "points": 78, "cast": ["Chris Evans", "Scarlett Johansson"] }+{"title": "Quantum Quest: A Cassini Space Odyssey", "starring": "Samuel L. Jackson", "points": 52, "cast": ["Chris Pine"] }+{"title": "Scott Pilgrim vs. the World", "starring": "Michael Cera", "points": 75, "cast": ["Alison Pill"] }+{"title": "Homeland Security", "starring": "Scott Glenn", "points": 43, "cast": ["Tom Skerritt"] }+{"title": "The Paperboy", "starring": "Scott Glenn", "points": 58, "cast": ["Zac Efron", "Matthew McConaughey"] }+{"title": "The Silence of the Lambs", "starring": "Scott Glenn", "points": 86, "cast": ["Jodie Foster"] }+{"title": "Confessions of a Shopaholic", "starring": "Kristin Scott Thomas", "points": 59, "cast": ["Isla Fisher", "Hugh Dancy"] }+{"title": "The Woman in the Fifth", "starring": "Kristin Scott Thomas", "points": 53, "cast": ["Ethan Hawke"] }+{"title": "Odd Thomas", "starring": "Matthew Page", "points": 69, "cast": ["Anton Yelchin", "Ashley Sommers"] }+{"title": "Suffering Man's Charity", "starring": "Henry Thomas", "points": 69, "cast": ["Alan Cumming", "Alison Guh", "David Boreanaz"] }+{"title": "The Gospel According to St. Matthew", "starring": "Paola Tedesco", "points": 79, "cast": ["Enrique Irazoqui", "Margherita Caruso"] }+{"title": "Halloween 5: The Revenge of Michael Myers", "starring": "Donald Pleasence", "points": 52, "cast": ["Danielle Harris", "Ellie Cornell"] }\ No newline at end of file