Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Merge branch 'v0.25-join' into v0.26-facets

Kishore Nallan5932a8d37d90
diff --cc include/collection.h
index d4b6212e,abe812e6..3ea53fdc
--- a/include/collection.h
+++ b/include/collection.h
@@@ -464,11 -466,8 +466,10 @@@ public
const size_t facet_sample_percent = 100,
const size_t facet_sample_threshold = 0,
const size_t page_offset = 0,
+ facet_index_type_t facet_index_type = HASH,
- const size_t vector_query_hits = 250,
const size_t remote_embedding_timeout_ms = 30000,
- const size_t remote_embedding_num_try = 2) const;
+ const size_t remote_embedding_num_try = 2,
+ const std::string& stopwords_set="") const;
Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result) const;
diff --cc include/topster.h
index d934a0a0,b0b8f125..1e7a91bc
--- a/include/topster.h
+++ b/include/topster.h
@@@ -15,11 -14,13 +15,11 @@@ struct KV
uint64_t key{};
uint64_t distinct_key{};
int64_t scores[3]{}; // match score + 2 custom attributes
-
+
// only to be used in hybrid search
- float vector_distance = 0.0f;
+ float vector_distance = 2.0f;
int64_t text_match_score = 0;
- reference_filter_result_t* reference_filter_result = nullptr;
-
// to be used only in final aggregation
uint64_t* query_indices = nullptr;
diff --cc src/collection.cpp
index 8973f374,8036da34..fa98fdd1
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@@ -1109,11 -1116,8 +1117,10 @@@ Option<nlohmann::json> Collection::sear
const size_t facet_sample_percent,
const size_t facet_sample_threshold,
const size_t page_offset,
+ facet_index_type_t facet_index_type,
- const size_t vector_query_hits,
const size_t remote_embedding_timeout_ms,
- const size_t remote_embedding_num_try) const {
+ const size_t remote_embedding_num_try,
+ const std::string& stopwords_set) const {
std::shared_lock lock(mutex);
diff --cc src/collection_manager.cpp
index a9c9b559,1c7e912b..9944b62c
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@@ -1158,11 -1079,8 +1155,10 @@@ Option<bool> CollectionManager::do_sear
facet_sample_percent,
facet_sample_threshold,
offset,
+ HASH,
- vector_query_hits,
remote_embedding_timeout_ms,
- remote_embedding_num_try
+ remote_embedding_num_try,
+ stopwords_set
);
uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(
diff --cc test/collection_filtering_test.cpp
index b3a5e600,cba5b21d..159989c9
--- a/test/collection_filtering_test.cpp
+++ b/test/collection_filtering_test.cpp
@@@ -2134,44 -2652,41 +2156,83 @@@ TEST_F(CollectionFilteringTest, Complex
collectionManager.drop_collection("ComplexFilterQueryCollection");
}
+TEST_F(CollectionFilteringTest, PrefixSearchWithFilter) {
+ std::ifstream infile(std::string(ROOT_DIR)+"test/documents.jsonl");
+ std::vector<field> search_fields = {
+ field("title", field_types::STRING, false),
+ field("points", field_types::INT32, false)
+ };
+
+ query_fields = {"title"};
+ sort_fields = { sort_by(sort_field_const::text_match, "DESC"), sort_by("points", "DESC") };
+
+ auto collection = collectionManager.create_collection("collection", 4, search_fields, "points").get();
+
+ std::string json_line;
+
+ // dummy record for record id 0: to make the test record IDs to match with line numbers
+ json_line = "{\"points\":10,\"title\":\"z\"}";
+ collection->add(json_line);
+
+ while (std::getline(infile, json_line)) {
+ collection->add(json_line);
+ }
+
+ infile.close();
+
+ std::vector<std::string> facets;
+ auto results = collection->search("what ex", query_fields, "points: >10", facets, sort_fields, {0}, 10, 1, MAX_SCORE, {true}, 10,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10).get();
+ ASSERT_EQ(7, results["hits"].size());
+ std::vector<std::string> ids = {"6", "12", "19", "22", "13", "8", "15"};
+
+ for(size_t i = 0; i < results["hits"].size(); i++) {
+ nlohmann::json result = results["hits"].at(i);
+ std::string result_id = result["document"]["id"];
+ std::string id = ids.at(i);
+ ASSERT_STREQ(id.c_str(), result_id.c_str());
+ }
+
+ collectionManager.drop_collection("collection");
- }
++}
++
+ TEST_F(CollectionFilteringTest, LargeFilterToken) {
+ nlohmann::json json =
+ R"({
+ "name": "LargeFilterTokenCollection",
+ "fields": [
+ {"name": "uri", "type": "string"}
+ ],
+ "symbols_to_index": [
+ "/",
+ "-"
+ ]
+ })"_json;
+
+ auto op = collectionManager.create_collection(json);
+ ASSERT_TRUE(op.ok());
+ auto coll = op.get();
+
+ json.clear();
+ std::string token = "rade/aols/insolvenzrecht/persoenliche-risiken-fuer-organe-von-kapitalgesellschaften-gmbh-"
+ "geschaeftsfuehrer-ag-vorstand";
+ json["uri"] = token;
+ auto add_op = coll->add(json.dump());
+ ASSERT_TRUE(add_op.ok());
+
+ auto results = coll->search("*", query_fields, "", {}, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();
+ ASSERT_EQ(1, results["hits"].size());
+
+ results = coll->search("*", query_fields, "uri:" + token, {}, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();
+ ASSERT_EQ(1, results["hits"].size());
+
+ token.erase(100); // Max token length that's indexed is 100, we'll still get a match.
+ results = coll->search("*", query_fields, "uri:" + token, {}, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();
+ ASSERT_EQ(1, results["hits"].size());
+
+ token.erase(99);
+ results = coll->search("*", query_fields, "uri:" + token, {}, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();
+ ASSERT_EQ(0, results["hits"].size());
+ }