Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Test for multi field search.
Kishore Nallan342ed2bab219
diff --git a/TODO.md b/TODO.mdindex 0e0ccf8c..3602fb2a 100644--- a/TODO.md+++ b/TODO.md@@ -17,6 +17,10 @@- ~~collection_id should be int, not string~~- ~~API should return count~~- ~~Fix documents.jsonl path in tests~~+- ~~Multi field search tests~~+- Index and search on multi-valued field+- Restore records as well on restart (like for meta)+- storage key prefix should include collection name- UTF-8 support for fuzzy search- Facets- Filters@@ -25,18 +29,14 @@- string_utils::tokenize should not have max length- only last token should be prefix searched- art float search-- storage key prefix should include collection name- Minimum results should be a variable instead of blindly going with max_results- Benchmark with -ffast-math- Space sensitivity- Use bitmap index instead of forarray for doc list-- Multi field search tests- Throw errors when schema is broken-- Index and search on multi-valued field- Assumption that all tokens match for scoring is no longer true- Primary_rank_scores and secondary_rank_scores hashmaps should be combined- Proper logging-- Restore records as well on restart (like for meta)- clean special chars before indexing- d-ary heap?- range search for art_intdiff --git a/src/collection.cpp b/src/collection.cppindex c1d44311..0b08e9cd 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -180,7 +180,6 @@ void Collection::search_candidates(int & token_rank, std::vector<std::vector<artnlohmann::json Collection::search(std::string query, const std::vector<std::string> fields,const int num_typos, const size_t num_results,const token_ordering token_order, const bool prefix) {- Topster<100> topster;size_t num_found = 0;// Order of `fields` are used to rank results@@ -188,6 +187,7 @@ nlohmann::json Collection::search(std::string query, const std::vector<std::stristd::vector<std::pair<int, Topster<100>::KV>> field_order_kvs;for(int i = 0; i < fields.size(); i++) {+ Topster<100> topster;const std::string & field = fields[i];search(query, field, num_typos, num_results, topster, num_found, token_order, prefix);diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 4aec3638..47e30249 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -55,6 +55,7 @@ protected:TEST_F(CollectionTest, ExactSearchShouldBeStable) {nlohmann::json results = collection->search("the", search_fields, 0, 10);ASSERT_EQ(7, results["hits"].size());+ ASSERT_EQ(7, results["found"].get<int>());// For two documents of the same score, the larger doc_id appears firststd::vector<std::string> ids = {"1", "6", "foo", "13", "10", "8", "16"};@@ -212,11 +213,13 @@ TEST_F(CollectionTest, TypoTokenRankedByScoreAndFrequency) {// Check paginationresults = collection->search("loox", search_fields, 1, 1, FREQUENCY, false);+ ASSERT_EQ(3, results["found"].get<int>());ASSERT_EQ(1, results["hits"].size());std::string solo_id = results["hits"].at(0)["id"];ASSERT_STREQ("3", solo_id.c_str());results = collection->search("loox", search_fields, 1, 2, FREQUENCY, false);+ ASSERT_EQ(3, results["found"].get<int>());ASSERT_EQ(2, results["hits"].size());// Check total ordering@@ -297,12 +300,16 @@ TEST_F(CollectionTest, PrefixSearching) {}TEST_F(CollectionTest, MultipleFields) {- /*Collection *coll_mul_fields;+ Collection *coll_mul_fields;std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");std::vector<field> fields = {field("title", field_types::STRING), field("starring", field_types::STRING)};std::vector<std::string> rank_fields = {"points"};- coll_mul_fields = new Collection("/tmp/typesense_test/coll_mul_fields", "coll_mul_fields", fields, rank_fields);++ coll_mul_fields = collectionManager.get_collection("coll_mul_fields");+ if(coll_mul_fields == nullptr) {+ coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields, rank_fields);+ }std::string json_line;@@ -313,6 +320,32 @@ TEST_F(CollectionTest, MultipleFields) {infile.close();search_fields = {"title", "starring"};+ nlohmann::json results = coll_mul_fields->search("Will", search_fields, 0, 10, FREQUENCY, false);+ ASSERT_EQ(4, results["hits"].size());++ std::vector<std::string> ids = {"3", "2", "1", "0"};++ for(size_t i = 0; i < results.size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["id"];+ std::string id = ids.at(i);+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }++ // when "starring" takes higher priority than "title"++ search_fields = {"starring", "title"};+ results = coll_mul_fields->search("thomas", search_fields, 0, 10, FREQUENCY, false);+ ASSERT_EQ(4, results["hits"].size());++ ids = {"15", "14", "12", "13"};++ for(size_t i = 0; i < results.size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["id"];+ std::string id = ids.at(i);+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }- delete coll_mul_fields;*/+ delete coll_mul_fields;}\ No newline at end of filediff --git a/test/config.h.in b/test/config.h.indeleted file mode 100644index e69de29b..00000000diff --git a/test/multi_field_documents.jsonl b/test/multi_field_documents.jsonlindex ccade46e..9782e632 100644--- a/test/multi_field_documents.jsonl+++ b/test/multi_field_documents.jsonl@@ -1,18 +1,18 @@-{"title: "Wake Up, Ron Burgundy: The Lost Movie", "starring": "Will Ferrell", "points": 62 }-{"title: "Anchorman 2: The Legend Continues", "starring": "Will Ferrell", "points": 63 }-{"title: "There Will Be Blood", "starring": "Daniel Day-Lewis", "points": 81 }-{"title: "Good Will Hunting", "starring": "Robin Williams", "points": 83 }-{"title: "The Adventures of Huck Finn", "starring": "Ron Perlman", "points": 58 }-{"title: "Percy Jackson: Sea of Monsters", "starring": "Ron Perlman", "points": 59 }-{"title: "Captain America: The Winter Soldier", "starring": "Samuel L. Jackson", "points": 78 }-{"title: "Quantum Quest: A Cassini Space Odyssey", "starring": "Samuel L. Jackson", "points": 52 }-{"title: "Scott Pilgrim vs. the World", "starring": "Michael Cera", "points": 75 }-{"title: "Homeland Security", "starring": "Scott Glenn", "points": 43 }-{"title: "The Paperboy", "starring": "Scott Glenn", "points": 58 }-{"title: "The Silence of the Lambs", "starring": "Scott Glenn", "points": 86 }-{"title: "Confessions of a Shopaholic", "starring": "Kristin Scott Thomas", "points": 59 }-{"title: "The Woman in the Fifth", "starring": "Kristin Scott Thomas", "points": 53 }-{"title: "Odd Thomas", "starring": "Matthew Page", "points": 69 }-{"title: "Suffering Man's Charity", "starring": "Henry Thomas", "points": 48 }-{"title: "The Gospel According to St. Matthew", "starring": "Paola Tedesco", "points": 79 }-{"title: "Halloween 5: The Revenge of Michael Myers", "starring": "Donald Pleasence", "points": 52 }\ No newline at end of file+{"title": "Wake Up, Ron Burgundy: The Lost Movie", "starring": "Will Ferrell", "points": 62 }+{"title": "Anchorman 2: The Legend Continues", "starring": "Will Ferrell", "points": 63 }+{"title": "There Will Be Blood", "starring": "Daniel Day-Lewis", "points": 81 }+{"title": "Good Will Hunting", "starring": "Robin Williams", "points": 83 }+{"title": "The Adventures of Huck Finn", "starring": "Ron Perlman", "points": 58 }+{"title": "Percy Jackson: Sea of Monsters", "starring": "Ron Perlman", "points": 59 }+{"title": "Captain America: The Winter Soldier", "starring": "Samuel L. Jackson", "points": 78 }+{"title": "Quantum Quest: A Cassini Space Odyssey", "starring": "Samuel L. Jackson", "points": 52 }+{"title": "Scott Pilgrim vs. the World", "starring": "Michael Cera", "points": 75 }+{"title": "Homeland Security", "starring": "Scott Glenn", "points": 43 }+{"title": "The Paperboy", "starring": "Scott Glenn", "points": 58 }+{"title": "The Silence of the Lambs", "starring": "Scott Glenn", "points": 86 }+{"title": "Confessions of a Shopaholic", "starring": "Kristin Scott Thomas", "points": 59 }+{"title": "The Woman in the Fifth", "starring": "Kristin Scott Thomas", "points": 53 }+{"title": "Odd Thomas", "starring": "Matthew Page", "points": 69 }+{"title": "Suffering Man's Charity", "starring": "Henry Thomas", "points": 69 }+{"title": "The Gospel According to St. Matthew", "starring": "Paola Tedesco", "points": 79 }+{"title": "Halloween 5: The Revenge of Michael Myers", "starring": "Donald Pleasence", "points": 52 }\ No newline at end of file