Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Merge branch 'v0.25-join' into v0.26-facets
Kishore Nallanae597f40baef
diff --cc include/index.hindex 748c24e6,1d895c02..40589a3b--- a/include/index.h+++ b/include/index.h@@@ -561,13 -581,12 +564,14 @@@ privateconst std::string& token, uint32_t seq_id);void initialize_facet_indexes(const field& facet_field);--- static void batch_embed_fields(std::vector<index_record*>& documents,- const tsl::htrie_map<char, field>& embedding_fields,- const tsl::htrie_map<char, field> & search_schema, const size_t remote_embedding_batch_size = 200);- static void batch_embed_fields(std::vector<index_record*>& documents,++ std::vector<group_by_field_it_t> get_group_by_field_iterators(const std::vector<std::string>&, bool is_reverse=false) const;+++ static void batch_embed_fields(std::vector<index_record*>& documents,+ const tsl::htrie_map<char, field>& embedding_fields,+ const tsl::htrie_map<char, field> & search_schema, const size_t remote_embedding_batch_size = 200,+ const size_t remote_embedding_timeout_ms = 60000, const size_t remote_embedding_num_tries = 2);+public:// for limiting number of results on multiple candidates / query rewritesenum {TYPO_TOKENS_THRESHOLD = 1};@@@ -724,9 -737,8 +729,10 @@@const std::vector<char>& token_separators,const std::vector<char>& symbols_to_index,const bool do_validation, const size_t remote_embedding_batch_size = 200,- const bool generate_embeddings = true,- const size_t remote_embedding_timeout_ms = 60000, const size_t remote_embedding_num_tries = 2, const bool generate_embeddings = true,- const bool use_addition_fields = false, const tsl::htrie_map<char, field>& addition_fields = {});++ const size_t remote_embedding_timeout_ms = 60000,++ const size_t remote_embedding_num_tries = 2, const bool generate_embeddings = true,+ const bool use_addition_fields = false,- const tsl::htrie_map<char, field>& addition_fields = {});++ const tsl::htrie_map<char, field>& addition_fields = tsl::htrie_map<char, field>());void index_field_in_memory(const field& afield, std::vector<index_record>& iter_batch);@@@ -1008,15 -1021,7 +1014,17 @@@Option<bool> seq_ids_outside_top_k(const std::string& field_name, size_t k,std::vector<uint32_t>& outside_seq_ids);+ Option<bool> get_related_ids(const std::string& collection_name,+ const std::string& reference_helper_field_name,+ const uint32_t& seq_id, std::vector<uint32_t>& result) const;++ Option<uint32_t> get_sort_index_value_with_lock(const std::string& collection_name,+ const std::string& field_name,+ const uint32_t& seq_id) const;++ friend class filter_result_iterator_t;+++ void repair_hnsw_index();};template<class T>diff --cc src/collection.cppindex fecb2898,29a17688..02220990--- a/src/collection.cpp+++ b/src/collection.cpp@@@ -4289,10 -3990,9 +4294,11 @@@ Option<bool> Collection::batch_alter_daindex->remove(seq_id, rec.doc, del_fields, true);}}-+Index::batch_memory_index(index, iter_batch, default_sorting_field, search_schema, embedding_fields,- fallback_field_type, token_separators, symbols_to_index, true, 200,- fallback_field_type, token_separators, symbols_to_index, true, 200, 60000, 2, found_embedding_field, true, schema_additions);++ fallback_field_type, token_separators, symbols_to_index, true, 200, 60000, 2,+ found_embedding_field, true, schema_additions);++if(found_embedding_field) {for(auto& index_record : iter_batch) {if(index_record.indexed.ok()) {@@@ -5378,15 -4873,12 +5384,15 @@@ bool Collection::get_enable_nested_fielreturn enable_nested_fields;}-Option<bool> Collection::parse_facet(const std::string& facet_field, std::vector<facet>& facets) const{- const std::regex base_pattern(".+\\(.*\\)");- const std::regex range_pattern("[[a-z A-Z]+:\\[([+-]?([0-9]*[.])?[0-9]*)\\,\\s*([+-]?([0-9]*[.])?[0-9]*)\\]");-- if(facet_field.find(":") != std::string::npos) { //range based facet- if(!std::regex_match(facet_field, base_pattern)){+Option<bool> Collection::parse_facet(const std::string& facet_field, std::vector<facet>& facets) const {+ const std::regex base_pattern(".+\\(.*\\)");- const std::regex range_pattern("[[a-zA-Z]+:\\[([0-9]+)\\,\\s*([0-9]+)\\]");++ const std::regex range_pattern("[[a-z A-Z]+:\\[([+-]?([0-9]*[.])?[0-9]*)\\,\\s*([+-]?([0-9]*[.])?[0-9]*)\\]");+ const std::string _alpha = "_alpha";++ if ((facet_field.find(":") != std::string::npos)+ && (facet_field.find("sort") == std::string::npos)) { //range based facet++ if (!std::regex_match(facet_field, base_pattern)) {std::string error = "Facet range value is not valid.";return Option<bool>(400, error);}@@@ -5469,27 -4962,52 +5475,52 @@@auto pos3 = range.find("]");int64_t lower_range, upper_range;- auto lower_range_start = pos1 + 2;- auto lower_range_len = pos2 - lower_range_start;- auto upper_range_start = pos2 + 1;- auto upper_range_len = pos3 - upper_range_start;if(a_field.is_integer()) {- std::string lower_range_str = range.substr(lower_range_start, lower_range_len);+ auto start = pos1 + 2;+ auto end = pos2 - start;+ auto lower_range_str = range.substr(start, end);StringUtils::trim(lower_range_str);- lower_range = std::stoll(lower_range_str);- std::string upper_range_str = range.substr(upper_range_start, upper_range_len);+ if(lower_range_str.empty()) {+ lower_range = INT64_MIN;+ } else {+ lower_range = std::stoll(lower_range_str);+ }++ start = pos2 + 1;+ end = pos3 - start;+ auto upper_range_str = range.substr(start, end);StringUtils::trim(upper_range_str);- upper_range = std::stoll(upper_range_str);+ if(upper_range_str.empty()) {+ upper_range = INT64_MAX;+ } else {+ upper_range = std::stoll(upper_range_str);+ }} else {- float val = std::stof(range.substr(pos1 + 2, pos2));- lower_range = Index::float_to_int64_t(val);+ auto start = pos1 + 2;+ auto end = pos2 - start;+ auto lower_range_str = range.substr(start, end);+ StringUtils::trim(lower_range_str);+ if(lower_range_str.empty()) {+ lower_range = INT64_MIN;+ } else {+ float val = std::stof(lower_range_str);+ lower_range = Index::float_to_int64_t(val);+ }- val = std::stof(range.substr(pos2 + 1, pos3));- upper_range = Index::float_to_int64_t(val);+ start = pos2 + 1;+ end = pos3 - start;+ auto upper_range_str = range.substr(start, end);+ StringUtils::trim(upper_range_str);+ if(upper_range_str.empty()) {+ upper_range = INT64_MAX;+ } else {+ float val = std::stof(upper_range_str);+ upper_range = Index::float_to_int64_t(val);+ }}- tupVec.emplace_back(std::make_tuple(lower_range, upper_range, range_val));+ tupVec.emplace_back(lower_range, upper_range, range_val);}//sort the range values so that we can check continuitydiff --cc src/index.cppindex 1ba401c3,f883c127..e7b04d1d--- a/src/index.cpp+++ b/src/index.cpp@@@ -206,15 -206,9 +206,16 @@@ Index::~Index()delete seq_ids;for(auto& vec_index_kv: vector_index) {+ std::unique_lock lock(vec_index_kv.second->repair_m);delete vec_index_kv.second;}++ for(auto & name_tree: reference_index) {+ delete name_tree.second;+ name_tree.second = nullptr;+ }++ reference_index.clear();}int64_t Index::get_points_from_doc(const nlohmann::json &document, const std::string & default_sorting_field) {diff --cc test/collection_vector_search_test.cppindex 70e0a15b,72244efe..5181c76b--- a/test/collection_vector_search_test.cpp+++ b/test/collection_vector_search_test.cpp@@@ -2747,3 -2730,96 +2747,95 @@@ TEST_F(CollectionVectorTest, TestSearchASSERT_FALSE(search_result.ok());ASSERT_EQ("Field `vec` is marked as a non-indexed field in the schema.", search_result.error());}+-+ TEST_F(CollectionVectorTest, TestSemanticSearchAfterUpdate) {+ nlohmann::json schema = R"({+ "name": "test",+ "fields": [+ {+ "name": "name",+ "type": "string"+ },+ {+ "name": "embedding",+ "type": "float[]",+ "embed": {+ "from": [+ "name"+ ],+ "model_config": {+ "model_name": "ts/e5-small"+ }+ }+ }+ ]+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto collection_create_op = collectionManager.create_collection(schema);+ ASSERT_TRUE(collection_create_op.ok());++ auto coll = collection_create_op.get();++ auto add_op = coll->add(R"({+ "name": "soccer",+ "id": "0"+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());++ add_op = coll->add(R"({+ "name": "basketball",+ "id": "1"+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());++ add_op = coll->add(R"({+ "name": "typesense",+ "id": "2"+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());++ add_op = coll->add(R"({+ "name": "potato",+ "id": "3"+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());++ auto result = coll->search("*", {}, "", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,+ fallback,+ 4, {off}, 32767, 32767, 2,+ false, true, "embedding:([], id:0, k:1)");++ ASSERT_TRUE(result.ok());+ ASSERT_EQ(1, result.get()["hits"].size());+ ASSERT_EQ("basketball", result.get()["hits"][0]["document"]["name"]);++ auto update_op = coll->add(R"({+ "name": "onion",+ "id": "0"+ })"_json.dump(), index_operation_t::UPDATE, "0");++ ASSERT_TRUE(update_op.ok());++ result = coll->search("*", {}, "", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,+ fallback,+ 4, {off}, 32767, 32767, 2,+ false, true, "embedding:([], id:0, k:1)");++ ASSERT_TRUE(result.ok());+ ASSERT_EQ(1, result.get()["hits"].size());+ ASSERT_EQ("potato", result.get()["hits"][0]["document"]["name"]);-}++}