Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Index and search on multi-valued string field.

Kishore Nallan385e8cb7a25a
diff --git a/TODO.md b/TODO.md
index 126266ac..83126bd6 100644
--- a/TODO.md
+++ b/TODO.md
@@ -20,8 +20,10 @@
- ~~Multi field search tests~~
- ~~storage key prefix should include collection name~~
- Index and search on multi-valued field
+- Proper field for ranking tokens
+- Support nested fields via "."
- ~~Restore records as well on restart (like for meta)~~
-- drop collection should remove all records from the store
+- ~~drop collection should remove all records from the store~~
- Pagination parameter
- UTF-8 support for fuzzy search
- Facets
diff --git a/include/collection.h b/include/collection.h
index e6d2d3ba..4047d093 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -51,7 +51,9 @@ private:
void search_candidates(int & token_rank, std::vector<std::vector<art_leaf*>> & token_leaves, Topster<100> & topster,
size_t & total_results, size_t & num_found, const size_t & max_results);
- void index_string_field(const std::string &field_name, art_tree *t, const nlohmann::json &document, uint32_t seq_id) const;
+ void index_string_field(const std::string & text, uint32_t score, art_tree *t, uint32_t seq_id) const;
+
+ void index_string_array_field(const std::string &field_name, art_tree *t, const nlohmann::json &document, uint32_t seq_id) const;
void index_numeric_field(const std::string &field_name, const std::string & field_type,
art_tree *t, const nlohmann::json &document, uint32_t seq_id) const;
diff --git a/include/field.h b/include/field.h
index f2701eb6..add92964 100644
--- a/include/field.h
+++ b/include/field.h
@@ -6,6 +6,7 @@ namespace field_types {
static const std::string STRING = "STRING";
static const std::string INT32 = "INT32";
static const std::string INT64 = "INT64";
+ static const std::string STRING_ARR = "STRING_ARR";
}
namespace fields {
diff --git a/src/collection.cpp b/src/collection.cpp
index 2499fe46..f5578006 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -54,9 +54,13 @@ void Collection::index_in_memory(const nlohmann::json &document, uint32_t seq_id
art_tree *t = index_map.at(field_name);
if(field_pair.second.type == field_types::STRING) {
- index_string_field(field_name, t, document, seq_id);
+ const std::string & text = document[field_name];
+ uint32_t points = document["points"];
+ index_string_field(text, points, t, seq_id);
} else if(field_pair.second.type == field_types::INT32 || field_pair.second.type == field_types::INT64) {
index_numeric_field(field_name, field_pair.second.type, t, document, seq_id);
+ } else if(field_pair.second.type == field_types::STRING_ARR) {
+ index_string_array_field(field_name, t, document, seq_id);
}
}
if(rank_fields.size() > 0 && document.count(rank_fields[0])) {
@@ -97,9 +101,7 @@ void Collection::index_numeric_field(const std::string &field_name, const std::s
art_insert(t, key, KEY_LEN, &art_doc, num_hits);
}
-void Collection::index_string_field(const std::string &field_name, art_tree *t, const nlohmann::json &document,
- uint32_t seq_id) const {
- std::string text = document[field_name];
+void Collection::index_string_field(const std::string & text, uint32_t score, art_tree *t, uint32_t seq_id) const {
std::vector<std::string> tokens;
StringUtils::tokenize(text, tokens, " ", true);
@@ -113,7 +115,7 @@ void Collection::index_string_field(const std::string &field_name, art_tree *t,
for(auto & kv: token_to_offsets) {
art_document art_doc;
art_doc.id = seq_id;
- art_doc.score = document["points"];
+ art_doc.score = score;
art_doc.offsets_len = (uint32_t) kv.second.size();
art_doc.offsets = new uint32_t[kv.second.size()];
@@ -138,6 +140,15 @@ void Collection::index_string_field(const std::string &field_name, art_tree *t,
}
}
+void Collection::index_string_array_field(const std::string &field_name, art_tree *t, const nlohmann::json &document,
+ uint32_t seq_id) const {
+ uint32_t points = document["points"];
+ std::vector<std::string> strings = document[field_name];
+ for(const std::string & str: strings) {
+ index_string_field(str, points, t, seq_id);
+ }
+}
+
void Collection::search_candidates(int & token_rank, std::vector<std::vector<art_leaf*>> & token_leaves,
Topster<100> & topster, size_t & total_results, size_t & num_found, const size_t & max_results) {
const size_t combination_limit = 10;
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 04b0f7a0..d5fbdf76 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -61,7 +61,7 @@ TEST_F(CollectionTest, ExactSearchShouldBeStable) {
// For two documents of the same score, the larger doc_id appears first
std::vector<std::string> ids = {"1", "6", "foo", "13", "10", "8", "16"};
- for(size_t i = 0; i < results.size(); i++) {
+ for(size_t i = 0; i < results["hits"].size(); i++) {
nlohmann::json result = results["hits"].at(i);
std::string id = ids.at(i);
std::string result_id = result["id"];
@@ -84,7 +84,7 @@ TEST_F(CollectionTest, ExactPhraseSearch) {
std::vector<std::string> ids = {"8", "1", "17", "16", "13"};
- for(size_t i = 0; i < results.size(); i++) {
+ for(size_t i = 0; i < results["hits"].size(); i++) {
nlohmann::json result = results["hits"].at(i);
std::string id = ids.at(i);
std::string result_id = result["id"];
@@ -109,7 +109,7 @@ TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {
std::vector<std::string> ids = {"2", "17"};
- for(size_t i = 0; i < results.size(); i++) {
+ for(size_t i = 0; i < results["hits"].size(); i++) {
nlohmann::json result = results["hits"].at(i);
std::string id = ids.at(i);
std::string result_id = result["id"];
@@ -120,7 +120,7 @@ TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {
results = collection->search("DoesNotExist from", search_fields, 1, 10);
ASSERT_EQ(2, results["hits"].size());
- for(size_t i = 0; i < results.size(); i++) {
+ for(size_t i = 0; i < results["hits"].size(); i++) {
nlohmann::json result = results["hits"].at(i);
std::string id = ids.at(i);
std::string result_id = result["id"];
@@ -132,7 +132,7 @@ TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {
ASSERT_EQ(2, results["hits"].size());
ids = {"2", "17"};
- for(size_t i = 0; i < results.size(); i++) {
+ for(size_t i = 0; i < results["hits"].size(); i++) {
nlohmann::json result = results["hits"].at(i);
std::string id = ids.at(i);
std::string result_id = result["id"];
@@ -154,7 +154,7 @@ TEST_F(CollectionTest, PartialPhraseSearch) {
std::vector<std::string> ids = {"1", "8", "16", "17"};
- for(size_t i = 0; i < results.size(); i++) {
+ for(size_t i = 0; i < results["hits"].size(); i++) {
nlohmann::json result = results["hits"].at(i);
std::string result_id = result["id"];
std::string id = ids.at(i);
@@ -168,7 +168,7 @@ TEST_F(CollectionTest, QueryWithTypo) {
std::vector<std::string> ids = {"19", "20", "21"};
- for(size_t i = 0; i < results.size(); i++) {
+ for(size_t i = 0; i < results["hits"].size(); i++) {
nlohmann::json result = results["hits"].at(i);
std::string result_id = result["id"];
std::string id = ids.at(i);
@@ -181,7 +181,7 @@ TEST_F(CollectionTest, QueryWithTypo) {
ASSERT_EQ(3, results["hits"].size());
- for(size_t i = 0; i < results.size(); i++) {
+ for(size_t i = 0; i < results["hits"].size(); i++) {
nlohmann::json result = results["hits"].at(i);
std::string result_id = result["id"];
std::string id = ids.at(i);
@@ -194,7 +194,7 @@ TEST_F(CollectionTest, TypoTokenRankedByScoreAndFrequency) {
ASSERT_EQ(2, results["hits"].size());
std::vector<std::string> ids = {"22", "23"};
- for(size_t i = 0; i < results.size(); i++) {
+ for(size_t i = 0; i < results["hits"].size(); i++) {
nlohmann::json result = results["hits"].at(i);
std::string result_id = result["id"];
std::string id = ids.at(i);
@@ -205,7 +205,7 @@ TEST_F(CollectionTest, TypoTokenRankedByScoreAndFrequency) {
ASSERT_EQ(3, results["hits"].size());
ids = {"3", "12", "24"};
- for(size_t i = 0; i < results.size(); i++) {
+ for(size_t i = 0; i < results["hits"].size(); i++) {
nlohmann::json result = results["hits"].at(i);
std::string result_id = result["id"];
std::string id = ids.at(i);
@@ -229,7 +229,7 @@ TEST_F(CollectionTest, TypoTokenRankedByScoreAndFrequency) {
ASSERT_EQ(5, results["hits"].size());
ids = {"3", "12", "24", "22", "23"};
- for(size_t i = 0; i < results.size(); i++) {
+ for(size_t i = 0; i < results["hits"].size(); i++) {
nlohmann::json result = results["hits"].at(i);
std::string result_id = result["id"];
std::string id = ids.at(i);
@@ -240,7 +240,7 @@ TEST_F(CollectionTest, TypoTokenRankedByScoreAndFrequency) {
ASSERT_EQ(5, results["hits"].size());
ids = {"22", "23", "3", "12", "24"};
- for(size_t i = 0; i < results.size(); i++) {
+ for(size_t i = 0; i < results["hits"].size(); i++) {
nlohmann::json result = results["hits"].at(i);
std::string result_id = result["id"];
std::string id = ids.at(i);
@@ -255,7 +255,7 @@ TEST_F(CollectionTest, TextContainingAnActualTypo) {
std::vector<std::string> ids = {"19", "6", "21", "8"};
- for(size_t i = 0; i < results.size(); i++) {
+ for(size_t i = 0; i < results["hits"].size(); i++) {
nlohmann::json result = results["hits"].at(i);
std::string result_id = result["id"];
std::string id = ids.at(i);
@@ -268,7 +268,7 @@ TEST_F(CollectionTest, TextContainingAnActualTypo) {
ids = {"20", "19", "6", "3", "21", "4", "10", "8"};
- for(size_t i = 0; i < results.size(); i++) {
+ for(size_t i = 0; i < results["hits"].size(); i++) {
nlohmann::json result = results["hits"].at(i);
std::string result_id = result["id"];
std::string id = ids.at(i);
@@ -281,7 +281,7 @@ TEST_F(CollectionTest, PrefixSearching) {
ASSERT_EQ(2, results["hits"].size());
std::vector<std::string> ids = {"12", "6"};
- for(size_t i = 0; i < results.size(); i++) {
+ for(size_t i = 0; i < results["hits"].size(); i++) {
nlohmann::json result = results["hits"].at(i);
std::string result_id = result["id"];
std::string id = ids.at(i);
@@ -292,7 +292,7 @@ TEST_F(CollectionTest, PrefixSearching) {
ASSERT_EQ(2, results["hits"].size());
ids = {"6", "12"};
- for(size_t i = 0; i < results.size(); i++) {
+ for(size_t i = 0; i < results["hits"].size(); i++) {
nlohmann::json result = results["hits"].at(i);
std::string result_id = result["id"];
std::string id = ids.at(i);
@@ -304,7 +304,8 @@ TEST_F(CollectionTest, MultipleFields) {
Collection *coll_mul_fields;
std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");
- std::vector<field> fields = {field("title", field_types::STRING), field("starring", field_types::STRING)};
+ std::vector<field> fields = {field("title", field_types::STRING), field("starring", field_types::STRING),
+ field("cast", field_types::STRING_ARR)};
std::vector<std::string> rank_fields = {"points"};
coll_mul_fields = collectionManager.get_collection("coll_mul_fields");
@@ -326,7 +327,7 @@ TEST_F(CollectionTest, MultipleFields) {
std::vector<std::string> ids = {"3", "2", "1", "0"};
- for(size_t i = 0; i < results.size(); i++) {
+ for(size_t i = 0; i < results["hits"].size(); i++) {
nlohmann::json result = results["hits"].at(i);
std::string result_id = result["id"];
std::string id = ids.at(i);
@@ -341,7 +342,35 @@ TEST_F(CollectionTest, MultipleFields) {
ids = {"15", "14", "12", "13"};
- for(size_t i = 0; i < results.size(); i++) {
+ for(size_t i = 0; i < results["hits"].size(); i++) {
+ nlohmann::json result = results["hits"].at(i);
+ std::string result_id = result["id"];
+ std::string id = ids.at(i);
+ ASSERT_STREQ(id.c_str(), result_id.c_str());
+ }
+
+ search_fields = {"starring", "title", "cast"};
+ results = coll_mul_fields->search("ben affleck", search_fields, 0, 10, FREQUENCY, false);
+ ASSERT_EQ(1, results["hits"].size());
+
+ search_fields = {"cast"};
+ results = coll_mul_fields->search("chris", search_fields, 0, 10, FREQUENCY, false);
+ ASSERT_EQ(3, results["hits"].size());
+
+ ids = {"6", "1", "7"};
+ for(size_t i = 0; i < results["hits"].size(); i++) {
+ nlohmann::json result = results["hits"].at(i);
+ std::string result_id = result["id"];
+ std::string id = ids.at(i);
+ ASSERT_STREQ(id.c_str(), result_id.c_str());
+ }
+
+ search_fields = {"cast"};
+ results = coll_mul_fields->search("chris pine", search_fields, 0, 10, FREQUENCY, false);
+ ASSERT_EQ(3, results["hits"].size());
+
+ ids = {"7", "6", "1"};
+ for(size_t i = 0; i < results["hits"].size(); i++) {
nlohmann::json result = results["hits"].at(i);
std::string result_id = result["id"];
std::string id = ids.at(i);
diff --git a/test/multi_field_documents.jsonl b/test/multi_field_documents.jsonl
index 9782e632..71a7bda8 100644
--- a/test/multi_field_documents.jsonl
+++ b/test/multi_field_documents.jsonl
@@ -1,18 +1,18 @@
-{"title": "Wake Up, Ron Burgundy: The Lost Movie", "starring": "Will Ferrell", "points": 62 }
-{"title": "Anchorman 2: The Legend Continues", "starring": "Will Ferrell", "points": 63 }
-{"title": "There Will Be Blood", "starring": "Daniel Day-Lewis", "points": 81 }
-{"title": "Good Will Hunting", "starring": "Robin Williams", "points": 83 }
-{"title": "The Adventures of Huck Finn", "starring": "Ron Perlman", "points": 58 }
-{"title": "Percy Jackson: Sea of Monsters", "starring": "Ron Perlman", "points": 59 }
-{"title": "Captain America: The Winter Soldier", "starring": "Samuel L. Jackson", "points": 78 }
-{"title": "Quantum Quest: A Cassini Space Odyssey", "starring": "Samuel L. Jackson", "points": 52 }
-{"title": "Scott Pilgrim vs. the World", "starring": "Michael Cera", "points": 75 }
-{"title": "Homeland Security", "starring": "Scott Glenn", "points": 43 }
-{"title": "The Paperboy", "starring": "Scott Glenn", "points": 58 }
-{"title": "The Silence of the Lambs", "starring": "Scott Glenn", "points": 86 }
-{"title": "Confessions of a Shopaholic", "starring": "Kristin Scott Thomas", "points": 59 }
-{"title": "The Woman in the Fifth", "starring": "Kristin Scott Thomas", "points": 53 }
-{"title": "Odd Thomas", "starring": "Matthew Page", "points": 69 }
-{"title": "Suffering Man's Charity", "starring": "Henry Thomas", "points": 69 }
-{"title": "The Gospel According to St. Matthew", "starring": "Paola Tedesco", "points": 79 }
-{"title": "Halloween 5: The Revenge of Michael Myers", "starring": "Donald Pleasence", "points": 52 }
\ No newline at end of file
+{"title": "Wake Up, Ron Burgundy: The Lost Movie", "starring": "Will Ferrell", "points": 62, "cast": [ "Adam McKay", "Steve Carell", "Paul Rudd"] }
+{"title": "Anchorman 2: The Legend Continues", "starring": "Will Ferrell", "points": 63, "cast": ["Josh Lawson", "Chris Parnell"] }
+{"title": "There Will Be Blood", "starring": "Daniel Day-Lewis", "points": 81, "cast": ["Martin Stringer", "Jacob Stringer", "Harrison Taylor"] }
+{"title": "Good Will Hunting", "starring": "Robin Williams", "points": 83, "cast": ["Matt Damon", "Ben Affleck", "Minnie Driver"] }
+{"title": "The Adventures of Huck Finn", "starring": "Ron Perlman", "points": 58, "cast": ["Elijah Wood", "Courtney B. Vance", "Robbie Coltrane"] }
+{"title": "Percy Jackson: Sea of Monsters", "starring": "Ron Perlman", "points": 59, "cast": ["Logan Lerman", "Alexandra Daddario"] }
+{"title": "Captain America: The Winter Soldier", "starring": "Samuel L. Jackson", "points": 78, "cast": ["Chris Evans", "Scarlett Johansson"] }
+{"title": "Quantum Quest: A Cassini Space Odyssey", "starring": "Samuel L. Jackson", "points": 52, "cast": ["Chris Pine"] }
+{"title": "Scott Pilgrim vs. the World", "starring": "Michael Cera", "points": 75, "cast": ["Alison Pill"] }
+{"title": "Homeland Security", "starring": "Scott Glenn", "points": 43, "cast": ["Tom Skerritt"] }
+{"title": "The Paperboy", "starring": "Scott Glenn", "points": 58, "cast": ["Zac Efron", "Matthew McConaughey"] }
+{"title": "The Silence of the Lambs", "starring": "Scott Glenn", "points": 86, "cast": ["Jodie Foster"] }
+{"title": "Confessions of a Shopaholic", "starring": "Kristin Scott Thomas", "points": 59, "cast": ["Isla Fisher", "Hugh Dancy"] }
+{"title": "The Woman in the Fifth", "starring": "Kristin Scott Thomas", "points": 53, "cast": ["Ethan Hawke"] }
+{"title": "Odd Thomas", "starring": "Matthew Page", "points": 69, "cast": ["Anton Yelchin", "Ashley Sommers"] }
+{"title": "Suffering Man's Charity", "starring": "Henry Thomas", "points": 69, "cast": ["Alan Cumming", "Alison Guh", "David Boreanaz"] }
+{"title": "The Gospel According to St. Matthew", "starring": "Paola Tedesco", "points": 79, "cast": ["Enrique Irazoqui", "Margherita Caruso"] }
+{"title": "Halloween 5: The Revenge of Michael Myers", "starring": "Donald Pleasence", "points": 52, "cast": ["Danielle Harris", "Ellie Cornell"] }
\ No newline at end of file