Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Merge branch 'v0.25-join' into v0.26-facets

Kishore Nallanae597f40baef
diff --cc include/index.h
index 748c24e6,1d895c02..40589a3b
--- a/include/index.h
+++ b/include/index.h
@@@ -561,13 -581,12 +564,14 @@@ private
const std::string& token, uint32_t seq_id);
void initialize_facet_indexes(const field& facet_field);
--
- static void batch_embed_fields(std::vector<index_record*>& documents,
- const tsl::htrie_map<char, field>& embedding_fields,
- const tsl::htrie_map<char, field> & search_schema, const size_t remote_embedding_batch_size = 200);
- static void batch_embed_fields(std::vector<index_record*>& documents,
+
+ std::vector<group_by_field_it_t> get_group_by_field_iterators(const std::vector<std::string>&, bool is_reverse=false) const;
+
++ static void batch_embed_fields(std::vector<index_record*>& documents,
+ const tsl::htrie_map<char, field>& embedding_fields,
+ const tsl::htrie_map<char, field> & search_schema, const size_t remote_embedding_batch_size = 200,
+ const size_t remote_embedding_timeout_ms = 60000, const size_t remote_embedding_num_tries = 2);
+
public:
// for limiting number of results on multiple candidates / query rewrites
enum {TYPO_TOKENS_THRESHOLD = 1};
@@@ -724,9 -737,8 +729,10 @@@
const std::vector<char>& token_separators,
const std::vector<char>& symbols_to_index,
const bool do_validation, const size_t remote_embedding_batch_size = 200,
- const bool generate_embeddings = true,
- const size_t remote_embedding_timeout_ms = 60000, const size_t remote_embedding_num_tries = 2, const bool generate_embeddings = true,
- const bool use_addition_fields = false, const tsl::htrie_map<char, field>& addition_fields = {});
++ const size_t remote_embedding_timeout_ms = 60000,
++ const size_t remote_embedding_num_tries = 2, const bool generate_embeddings = true,
+ const bool use_addition_fields = false,
- const tsl::htrie_map<char, field>& addition_fields = {});
++ const tsl::htrie_map<char, field>& addition_fields = tsl::htrie_map<char, field>());
void index_field_in_memory(const field& afield, std::vector<index_record>& iter_batch);
@@@ -1008,15 -1021,7 +1014,17 @@@
Option<bool> seq_ids_outside_top_k(const std::string& field_name, size_t k,
std::vector<uint32_t>& outside_seq_ids);
+ Option<bool> get_related_ids(const std::string& collection_name,
+ const std::string& reference_helper_field_name,
+ const uint32_t& seq_id, std::vector<uint32_t>& result) const;
+
+ Option<uint32_t> get_sort_index_value_with_lock(const std::string& collection_name,
+ const std::string& field_name,
+ const uint32_t& seq_id) const;
+
+ friend class filter_result_iterator_t;
++
+ void repair_hnsw_index();
};
template<class T>
diff --cc src/collection.cpp
index fecb2898,29a17688..02220990
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@@ -4289,10 -3990,9 +4294,11 @@@ Option<bool> Collection::batch_alter_da
index->remove(seq_id, rec.doc, del_fields, true);
}
}
-
+
Index::batch_memory_index(index, iter_batch, default_sorting_field, search_schema, embedding_fields,
- fallback_field_type, token_separators, symbols_to_index, true, 200,
- fallback_field_type, token_separators, symbols_to_index, true, 200, 60000, 2, found_embedding_field, true, schema_additions);
++ fallback_field_type, token_separators, symbols_to_index, true, 200, 60000, 2,
+ found_embedding_field, true, schema_additions);
++
if(found_embedding_field) {
for(auto& index_record : iter_batch) {
if(index_record.indexed.ok()) {
@@@ -5378,15 -4873,12 +5384,15 @@@ bool Collection::get_enable_nested_fiel
return enable_nested_fields;
}
-Option<bool> Collection::parse_facet(const std::string& facet_field, std::vector<facet>& facets) const{
- const std::regex base_pattern(".+\\(.*\\)");
- const std::regex range_pattern("[[a-z A-Z]+:\\[([+-]?([0-9]*[.])?[0-9]*)\\,\\s*([+-]?([0-9]*[.])?[0-9]*)\\]");
-
- if(facet_field.find(":") != std::string::npos) { //range based facet
- if(!std::regex_match(facet_field, base_pattern)){
+Option<bool> Collection::parse_facet(const std::string& facet_field, std::vector<facet>& facets) const {
+ const std::regex base_pattern(".+\\(.*\\)");
- const std::regex range_pattern("[[a-zA-Z]+:\\[([0-9]+)\\,\\s*([0-9]+)\\]");
++ const std::regex range_pattern("[[a-z A-Z]+:\\[([+-]?([0-9]*[.])?[0-9]*)\\,\\s*([+-]?([0-9]*[.])?[0-9]*)\\]");
+ const std::string _alpha = "_alpha";
+
+ if ((facet_field.find(":") != std::string::npos)
+ && (facet_field.find("sort") == std::string::npos)) { //range based facet
+
+ if (!std::regex_match(facet_field, base_pattern)) {
std::string error = "Facet range value is not valid.";
return Option<bool>(400, error);
}
@@@ -5469,27 -4962,52 +5475,52 @@@
auto pos3 = range.find("]");
int64_t lower_range, upper_range;
- auto lower_range_start = pos1 + 2;
- auto lower_range_len = pos2 - lower_range_start;
- auto upper_range_start = pos2 + 1;
- auto upper_range_len = pos3 - upper_range_start;
if(a_field.is_integer()) {
- std::string lower_range_str = range.substr(lower_range_start, lower_range_len);
+ auto start = pos1 + 2;
+ auto end = pos2 - start;
+ auto lower_range_str = range.substr(start, end);
StringUtils::trim(lower_range_str);
- lower_range = std::stoll(lower_range_str);
- std::string upper_range_str = range.substr(upper_range_start, upper_range_len);
+ if(lower_range_str.empty()) {
+ lower_range = INT64_MIN;
+ } else {
+ lower_range = std::stoll(lower_range_str);
+ }
+
+ start = pos2 + 1;
+ end = pos3 - start;
+ auto upper_range_str = range.substr(start, end);
StringUtils::trim(upper_range_str);
- upper_range = std::stoll(upper_range_str);
+ if(upper_range_str.empty()) {
+ upper_range = INT64_MAX;
+ } else {
+ upper_range = std::stoll(upper_range_str);
+ }
} else {
- float val = std::stof(range.substr(pos1 + 2, pos2));
- lower_range = Index::float_to_int64_t(val);
+ auto start = pos1 + 2;
+ auto end = pos2 - start;
+ auto lower_range_str = range.substr(start, end);
+ StringUtils::trim(lower_range_str);
+ if(lower_range_str.empty()) {
+ lower_range = INT64_MIN;
+ } else {
+ float val = std::stof(lower_range_str);
+ lower_range = Index::float_to_int64_t(val);
+ }
- val = std::stof(range.substr(pos2 + 1, pos3));
- upper_range = Index::float_to_int64_t(val);
+ start = pos2 + 1;
+ end = pos3 - start;
+ auto upper_range_str = range.substr(start, end);
+ StringUtils::trim(upper_range_str);
+ if(upper_range_str.empty()) {
+ upper_range = INT64_MAX;
+ } else {
+ float val = std::stof(upper_range_str);
+ upper_range = Index::float_to_int64_t(val);
+ }
}
- tupVec.emplace_back(std::make_tuple(lower_range, upper_range, range_val));
+ tupVec.emplace_back(lower_range, upper_range, range_val);
}
//sort the range values so that we can check continuity
diff --cc src/index.cpp
index 1ba401c3,f883c127..e7b04d1d
--- a/src/index.cpp
+++ b/src/index.cpp
@@@ -206,15 -206,9 +206,16 @@@ Index::~Index()
delete seq_ids;
for(auto& vec_index_kv: vector_index) {
+ std::unique_lock lock(vec_index_kv.second->repair_m);
delete vec_index_kv.second;
}
+
+ for(auto & name_tree: reference_index) {
+ delete name_tree.second;
+ name_tree.second = nullptr;
+ }
+
+ reference_index.clear();
}
int64_t Index::get_points_from_doc(const nlohmann::json &document, const std::string & default_sorting_field) {
diff --cc test/collection_vector_search_test.cpp
index 70e0a15b,72244efe..5181c76b
--- a/test/collection_vector_search_test.cpp
+++ b/test/collection_vector_search_test.cpp
@@@ -2747,3 -2730,96 +2747,95 @@@ TEST_F(CollectionVectorTest, TestSearch
ASSERT_FALSE(search_result.ok());
ASSERT_EQ("Field `vec` is marked as a non-indexed field in the schema.", search_result.error());
}
+
-
+ TEST_F(CollectionVectorTest, TestSemanticSearchAfterUpdate) {
+ nlohmann::json schema = R"({
+ "name": "test",
+ "fields": [
+ {
+ "name": "name",
+ "type": "string"
+ },
+ {
+ "name": "embedding",
+ "type": "float[]",
+ "embed": {
+ "from": [
+ "name"
+ ],
+ "model_config": {
+ "model_name": "ts/e5-small"
+ }
+ }
+ }
+ ]
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto collection_create_op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto coll = collection_create_op.get();
+
+ auto add_op = coll->add(R"({
+ "name": "soccer",
+ "id": "0"
+ })"_json.dump());
+
+ ASSERT_TRUE(add_op.ok());
+
+ add_op = coll->add(R"({
+ "name": "basketball",
+ "id": "1"
+ })"_json.dump());
+
+ ASSERT_TRUE(add_op.ok());
+
+ add_op = coll->add(R"({
+ "name": "typesense",
+ "id": "2"
+ })"_json.dump());
+
+ ASSERT_TRUE(add_op.ok());
+
+ add_op = coll->add(R"({
+ "name": "potato",
+ "id": "3"
+ })"_json.dump());
+
+ ASSERT_TRUE(add_op.ok());
+
+ auto result = coll->search("*", {}, "", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,
+ fallback,
+ 4, {off}, 32767, 32767, 2,
+ false, true, "embedding:([], id:0, k:1)");
+
+ ASSERT_TRUE(result.ok());
+ ASSERT_EQ(1, result.get()["hits"].size());
+ ASSERT_EQ("basketball", result.get()["hits"][0]["document"]["name"]);
+
+ auto update_op = coll->add(R"({
+ "name": "onion",
+ "id": "0"
+ })"_json.dump(), index_operation_t::UPDATE, "0");
+
+ ASSERT_TRUE(update_op.ok());
+
+ result = coll->search("*", {}, "", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,
+ fallback,
+ 4, {off}, 32767, 32767, 2,
+ false, true, "embedding:([], id:0, k:1)");
+
+ ASSERT_TRUE(result.ok());
+ ASSERT_EQ(1, result.get()["hits"].size());
+ ASSERT_EQ("potato", result.get()["hits"][0]["document"]["name"]);
-}
++}