Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Test for multi field search.

Kishore Nallan342ed2bab219
diff --git a/TODO.md b/TODO.md
index 0e0ccf8c..3602fb2a 100644
--- a/TODO.md
+++ b/TODO.md
@@ -17,6 +17,10 @@
- ~~collection_id should be int, not string~~
- ~~API should return count~~
- ~~Fix documents.jsonl path in tests~~
+- ~~Multi field search tests~~
+- Index and search on multi-valued field
+- Restore records as well on restart (like for meta)
+- storage key prefix should include collection name
- UTF-8 support for fuzzy search
- Facets
- Filters
@@ -25,18 +29,14 @@
- string_utils::tokenize should not have max length
- only last token should be prefix searched
- art float search
-- storage key prefix should include collection name
- Minimum results should be a variable instead of blindly going with max_results
- Benchmark with -ffast-math
- Space sensitivity
- Use bitmap index instead of forarray for doc list
-- Multi field search tests
- Throw errors when schema is broken
-- Index and search on multi-valued field
- Assumption that all tokens match for scoring is no longer true
- Primary_rank_scores and secondary_rank_scores hashmaps should be combined
- Proper logging
-- Restore records as well on restart (like for meta)
- clean special chars before indexing
- d-ary heap?
- range search for art_int
diff --git a/src/collection.cpp b/src/collection.cpp
index c1d44311..0b08e9cd 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -180,7 +180,6 @@ void Collection::search_candidates(int & token_rank, std::vector<std::vector<art
nlohmann::json Collection::search(std::string query, const std::vector<std::string> fields,
const int num_typos, const size_t num_results,
const token_ordering token_order, const bool prefix) {
- Topster<100> topster;
size_t num_found = 0;
// Order of `fields` are used to rank results
@@ -188,6 +187,7 @@ nlohmann::json Collection::search(std::string query, const std::vector<std::stri
std::vector<std::pair<int, Topster<100>::KV>> field_order_kvs;
for(int i = 0; i < fields.size(); i++) {
+ Topster<100> topster;
const std::string & field = fields[i];
search(query, field, num_typos, num_results, topster, num_found, token_order, prefix);
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 4aec3638..47e30249 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -55,6 +55,7 @@ protected:
TEST_F(CollectionTest, ExactSearchShouldBeStable) {
nlohmann::json results = collection->search("the", search_fields, 0, 10);
ASSERT_EQ(7, results["hits"].size());
+ ASSERT_EQ(7, results["found"].get<int>());
// For two documents of the same score, the larger doc_id appears first
std::vector<std::string> ids = {"1", "6", "foo", "13", "10", "8", "16"};
@@ -212,11 +213,13 @@ TEST_F(CollectionTest, TypoTokenRankedByScoreAndFrequency) {
// Check pagination
results = collection->search("loox", search_fields, 1, 1, FREQUENCY, false);
+ ASSERT_EQ(3, results["found"].get<int>());
ASSERT_EQ(1, results["hits"].size());
std::string solo_id = results["hits"].at(0)["id"];
ASSERT_STREQ("3", solo_id.c_str());
results = collection->search("loox", search_fields, 1, 2, FREQUENCY, false);
+ ASSERT_EQ(3, results["found"].get<int>());
ASSERT_EQ(2, results["hits"].size());
// Check total ordering
@@ -297,12 +300,16 @@ TEST_F(CollectionTest, PrefixSearching) {
}
TEST_F(CollectionTest, MultipleFields) {
- /*Collection *coll_mul_fields;
+ Collection *coll_mul_fields;
std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");
std::vector<field> fields = {field("title", field_types::STRING), field("starring", field_types::STRING)};
std::vector<std::string> rank_fields = {"points"};
- coll_mul_fields = new Collection("/tmp/typesense_test/coll_mul_fields", "coll_mul_fields", fields, rank_fields);
+
+ coll_mul_fields = collectionManager.get_collection("coll_mul_fields");
+ if(coll_mul_fields == nullptr) {
+ coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields, rank_fields);
+ }
std::string json_line;
@@ -313,6 +320,32 @@ TEST_F(CollectionTest, MultipleFields) {
infile.close();
search_fields = {"title", "starring"};
+ nlohmann::json results = coll_mul_fields->search("Will", search_fields, 0, 10, FREQUENCY, false);
+ ASSERT_EQ(4, results["hits"].size());
+
+ std::vector<std::string> ids = {"3", "2", "1", "0"};
+
+ for(size_t i = 0; i < results.size(); i++) {
+ nlohmann::json result = results["hits"].at(i);
+ std::string result_id = result["id"];
+ std::string id = ids.at(i);
+ ASSERT_STREQ(id.c_str(), result_id.c_str());
+ }
+
+ // when "starring" takes higher priority than "title"
+
+ search_fields = {"starring", "title"};
+ results = coll_mul_fields->search("thomas", search_fields, 0, 10, FREQUENCY, false);
+ ASSERT_EQ(4, results["hits"].size());
+
+ ids = {"15", "14", "12", "13"};
+
+ for(size_t i = 0; i < results.size(); i++) {
+ nlohmann::json result = results["hits"].at(i);
+ std::string result_id = result["id"];
+ std::string id = ids.at(i);
+ ASSERT_STREQ(id.c_str(), result_id.c_str());
+ }
- delete coll_mul_fields;*/
+ delete coll_mul_fields;
}
\ No newline at end of file
diff --git a/test/config.h.in b/test/config.h.in
deleted file mode 100644
index e69de29b..00000000
diff --git a/test/multi_field_documents.jsonl b/test/multi_field_documents.jsonl
index ccade46e..9782e632 100644
--- a/test/multi_field_documents.jsonl
+++ b/test/multi_field_documents.jsonl
@@ -1,18 +1,18 @@
-{"title: "Wake Up, Ron Burgundy: The Lost Movie", "starring": "Will Ferrell", "points": 62 }
-{"title: "Anchorman 2: The Legend Continues", "starring": "Will Ferrell", "points": 63 }
-{"title: "There Will Be Blood", "starring": "Daniel Day-Lewis", "points": 81 }
-{"title: "Good Will Hunting", "starring": "Robin Williams", "points": 83 }
-{"title: "The Adventures of Huck Finn", "starring": "Ron Perlman", "points": 58 }
-{"title: "Percy Jackson: Sea of Monsters", "starring": "Ron Perlman", "points": 59 }
-{"title: "Captain America: The Winter Soldier", "starring": "Samuel L. Jackson", "points": 78 }
-{"title: "Quantum Quest: A Cassini Space Odyssey", "starring": "Samuel L. Jackson", "points": 52 }
-{"title: "Scott Pilgrim vs. the World", "starring": "Michael Cera", "points": 75 }
-{"title: "Homeland Security", "starring": "Scott Glenn", "points": 43 }
-{"title: "The Paperboy", "starring": "Scott Glenn", "points": 58 }
-{"title: "The Silence of the Lambs", "starring": "Scott Glenn", "points": 86 }
-{"title: "Confessions of a Shopaholic", "starring": "Kristin Scott Thomas", "points": 59 }
-{"title: "The Woman in the Fifth", "starring": "Kristin Scott Thomas", "points": 53 }
-{"title: "Odd Thomas", "starring": "Matthew Page", "points": 69 }
-{"title: "Suffering Man's Charity", "starring": "Henry Thomas", "points": 48 }
-{"title: "The Gospel According to St. Matthew", "starring": "Paola Tedesco", "points": 79 }
-{"title: "Halloween 5: The Revenge of Michael Myers", "starring": "Donald Pleasence", "points": 52 }
\ No newline at end of file
+{"title": "Wake Up, Ron Burgundy: The Lost Movie", "starring": "Will Ferrell", "points": 62 }
+{"title": "Anchorman 2: The Legend Continues", "starring": "Will Ferrell", "points": 63 }
+{"title": "There Will Be Blood", "starring": "Daniel Day-Lewis", "points": 81 }
+{"title": "Good Will Hunting", "starring": "Robin Williams", "points": 83 }
+{"title": "The Adventures of Huck Finn", "starring": "Ron Perlman", "points": 58 }
+{"title": "Percy Jackson: Sea of Monsters", "starring": "Ron Perlman", "points": 59 }
+{"title": "Captain America: The Winter Soldier", "starring": "Samuel L. Jackson", "points": 78 }
+{"title": "Quantum Quest: A Cassini Space Odyssey", "starring": "Samuel L. Jackson", "points": 52 }
+{"title": "Scott Pilgrim vs. the World", "starring": "Michael Cera", "points": 75 }
+{"title": "Homeland Security", "starring": "Scott Glenn", "points": 43 }
+{"title": "The Paperboy", "starring": "Scott Glenn", "points": 58 }
+{"title": "The Silence of the Lambs", "starring": "Scott Glenn", "points": 86 }
+{"title": "Confessions of a Shopaholic", "starring": "Kristin Scott Thomas", "points": 59 }
+{"title": "The Woman in the Fifth", "starring": "Kristin Scott Thomas", "points": 53 }
+{"title": "Odd Thomas", "starring": "Matthew Page", "points": 69 }
+{"title": "Suffering Man's Charity", "starring": "Henry Thomas", "points": 69 }
+{"title": "The Gospel According to St. Matthew", "starring": "Paola Tedesco", "points": 79 }
+{"title": "Halloween 5: The Revenge of Michael Myers", "starring": "Donald Pleasence", "points": 52 }
\ No newline at end of file