Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Merge branch 'v0.25-join' into v0.26-facets
Kishore Nallan5932a8d37d90
diff --cc include/collection.hindex d4b6212e,abe812e6..3ea53fdc--- a/include/collection.h+++ b/include/collection.h@@@ -464,11 -466,8 +466,10 @@@ publicconst size_t facet_sample_percent = 100,const size_t facet_sample_threshold = 0,const size_t page_offset = 0,+ facet_index_type_t facet_index_type = HASH,- const size_t vector_query_hits = 250,const size_t remote_embedding_timeout_ms = 30000,- const size_t remote_embedding_num_try = 2) const;+ const size_t remote_embedding_num_try = 2,+ const std::string& stopwords_set="") const;Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result) const;diff --cc include/topster.hindex d934a0a0,b0b8f125..1e7a91bc--- a/include/topster.h+++ b/include/topster.h@@@ -15,11 -14,13 +15,11 @@@ struct KVuint64_t key{};uint64_t distinct_key{};int64_t scores[3]{}; // match score + 2 custom attributes-+// only to be used in hybrid search- float vector_distance = 0.0f;+ float vector_distance = 2.0f;int64_t text_match_score = 0;- reference_filter_result_t* reference_filter_result = nullptr;-// to be used only in final aggregationuint64_t* query_indices = nullptr;diff --cc src/collection.cppindex 8973f374,8036da34..fa98fdd1--- a/src/collection.cpp+++ b/src/collection.cpp@@@ -1109,11 -1116,8 +1117,10 @@@ Option<nlohmann::json> Collection::searconst size_t facet_sample_percent,const size_t facet_sample_threshold,const size_t page_offset,+ facet_index_type_t facet_index_type,- const size_t vector_query_hits,const size_t remote_embedding_timeout_ms,- const size_t remote_embedding_num_try) const {+ const size_t remote_embedding_num_try,+ const std::string& stopwords_set) const {std::shared_lock lock(mutex);diff --cc src/collection_manager.cppindex a9c9b559,1c7e912b..9944b62c--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@@ -1158,11 -1079,8 +1155,10 @@@ Option<bool> CollectionManager::do_searfacet_sample_percent,facet_sample_threshold,offset,+ HASH,- vector_query_hits,remote_embedding_timeout_ms,- remote_embedding_num_try+ remote_embedding_num_try,+ stopwords_set);uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(diff --cc test/collection_filtering_test.cppindex b3a5e600,cba5b21d..159989c9--- a/test/collection_filtering_test.cpp+++ b/test/collection_filtering_test.cpp@@@ -2134,44 -2652,41 +2156,83 @@@ TEST_F(CollectionFilteringTest, ComplexcollectionManager.drop_collection("ComplexFilterQueryCollection");}+TEST_F(CollectionFilteringTest, PrefixSearchWithFilter) {+ std::ifstream infile(std::string(ROOT_DIR)+"test/documents.jsonl");+ std::vector<field> search_fields = {+ field("title", field_types::STRING, false),+ field("points", field_types::INT32, false)+ };++ query_fields = {"title"};+ sort_fields = { sort_by(sort_field_const::text_match, "DESC"), sort_by("points", "DESC") };++ auto collection = collectionManager.create_collection("collection", 4, search_fields, "points").get();++ std::string json_line;++ // dummy record for record id 0: to make the test record IDs to match with line numbers+ json_line = "{\"points\":10,\"title\":\"z\"}";+ collection->add(json_line);++ while (std::getline(infile, json_line)) {+ collection->add(json_line);+ }++ infile.close();++ std::vector<std::string> facets;+ auto results = collection->search("what ex", query_fields, "points: >10", facets, sort_fields, {0}, 10, 1, MAX_SCORE, {true}, 10,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10).get();+ ASSERT_EQ(7, results["hits"].size());+ std::vector<std::string> ids = {"6", "12", "19", "22", "13", "8", "15"};++ for(size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["document"]["id"];+ std::string id = ids.at(i);+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }++ collectionManager.drop_collection("collection");- }++}+++ TEST_F(CollectionFilteringTest, LargeFilterToken) {+ nlohmann::json json =+ R"({+ "name": "LargeFilterTokenCollection",+ "fields": [+ {"name": "uri", "type": "string"}+ ],+ "symbols_to_index": [+ "/",+ "-"+ ]+ })"_json;++ auto op = collectionManager.create_collection(json);+ ASSERT_TRUE(op.ok());+ auto coll = op.get();++ json.clear();+ std::string token = "rade/aols/insolvenzrecht/persoenliche-risiken-fuer-organe-von-kapitalgesellschaften-gmbh-"+ "geschaeftsfuehrer-ag-vorstand";+ json["uri"] = token;+ auto add_op = coll->add(json.dump());+ ASSERT_TRUE(add_op.ok());++ auto results = coll->search("*", query_fields, "", {}, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(1, results["hits"].size());++ results = coll->search("*", query_fields, "uri:" + token, {}, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(1, results["hits"].size());++ token.erase(100); // Max token length that's indexed is 100, we'll still get a match.+ results = coll->search("*", query_fields, "uri:" + token, {}, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(1, results["hits"].size());++ token.erase(99);+ results = coll->search("*", query_fields, "uri:" + token, {}, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(0, results["hits"].size());+ }