Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Persist vector field properties and restore.
Kishore Nallan49268d648e7d
diff --git a/include/field.h b/include/field.hindex 5ceaf909..0ff3d7d4 100644--- a/include/field.h+++ b/include/field.h@@ -293,6 +293,16 @@ struct field {field_val[fields::locale] = field.locale;+ field_val[fields::nested] = field.nested;+ if(field.nested) {+ field_val[fields::nested_array] = field.nested_array;+ }++ if(field.num_dim > 0) {+ field_val[fields::num_dim] = field.num_dim;+ field_val[fields::vec_dist] = field.vec_dist == ip ? "ip" : "cosine";+ }+fields_json.push_back(field_val);if(!field.has_valid_type()) {diff --git a/src/collection.cpp b/src/collection.cppindex db0afa35..8ad5e553 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -168,6 +168,15 @@ nlohmann::json Collection::get_summary_json() const {field_json[fields::sort] = coll_field.sort;field_json[fields::infix] = coll_field.infix;field_json[fields::locale] = coll_field.locale;+ field_json[fields::nested] = coll_field.nested;++ if(coll_field.nested) {+ field_json[fields::nested_array] = coll_field.nested_array;+ }++ if(coll_field.num_dim > 0) {+ field_json[fields::num_dim] = coll_field.num_dim;+ }fields_arr.push_back(field_json);}diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex a85873b4..711ee406 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -42,9 +42,31 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collectionfield_obj[fields::infix] = -1;}+ if(field_obj.count(fields::nested) == 0) {+ field_obj[fields::nested] = false;+ }++ if(field_obj.count(fields::nested_array) == 0) {+ field_obj[fields::nested_array] = 0;+ }++ if(field_obj.count(fields::num_dim) == 0) {+ field_obj[fields::num_dim] = 0;+ }++ vector_distance_type_t vec_dist_type = vector_distance_type_t::cosine;++ if(field_obj.count(fields::vec_dist) != 0) {+ auto vec_dist_type_op = magic_enum::enum_cast<vector_distance_type_t>(fields::vec_dist);+ if(vec_dist_type_op.has_value()) {+ vec_dist_type = vec_dist_type_op.value();+ }+ }+field f(field_obj[fields::name], field_obj[fields::type], field_obj[fields::facet],field_obj[fields::optional], field_obj[fields::index], field_obj[fields::locale],- -1, field_obj[fields::infix]);+ -1, field_obj[fields::infix], field_obj[fields::nested], field_obj[fields::nested_array],+ field_obj[fields::num_dim], vec_dist_type);// value of `sort` depends on field typeif(field_obj.count(fields::sort) == 0) {diff --git a/src/index.cpp b/src/index.cppindex 355e2821..a10f1840 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -2576,6 +2576,8 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::vscores[0] = -float_to_int64_t(dist_label.first);int64_t match_score_index = -1;+ //LOG(INFO) << "SEQ_ID: " << seq_id << ", score: " << dist_label.first;+KV kv(0, searched_queries.size(), 0, seq_id, distinct_id, match_score_index, scores);topster->add(&kv);nearest_ids.push_back(seq_id);diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex 61d1f71d..a7fe2f07 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -12,8 +12,8 @@ protected:CollectionManager & collectionManager = CollectionManager::get_instance();std::atomic<bool> quit = false;Collection *collection1;- std::vector<field> search_fields;std::vector<sort_by> sort_fields;+ nlohmann::json schema;void setupCollection() {std::string state_dir_path = "/tmp/typesense_test/coll_manager_test_db";@@ -24,19 +24,27 @@ protected:collectionManager.init(store, 1.0, "auth_key", quit);collectionManager.load(8, 1000);- search_fields = {- field("title", field_types::STRING, false, false, true, "en", false),- field("starring", field_types::STRING, false, false, true, "", false, true),- field("cast", field_types::STRING_ARRAY, true, true, true, "", false),- field(".*_year", field_types::INT32, true, true),- field("location", field_types::GEOPOINT, false, true, true),- field("not_stored", field_types::STRING, false, true, false),- field("points", field_types::INT32, false)- };+ schema = R"({+ "name": "collection1",+ "enable_nested_fields": true,+ "fields": [+ {"name": "title", "type": "string", "locale": "en"},+ {"name": "starring", "type": "string", "infix": true},+ {"name": "cast", "type": "string[]", "facet": true, "optional": true},+ {"name": ".*_year", "type": "int32", "facet": true, "optional": true},+ {"name": "location", "type": "geopoint", "optional": true},+ {"name": "not_stored", "type": "string", "optional": true, "index": false},+ {"name": "points", "type": "int32"},+ {"name": "person", "type": "object", "optional": true},+ {"name": "vec", "type": "float[]", "num_dim": 128, "optional": true}+ ],+ "default_sorting_field": "points",+ "symbols_to_index":["+"],+ "token_separators":["-"]+ })"_json;sort_fields = { sort_by("points", "DESC") };- collection1 = collectionManager.create_collection("collection1", 4, search_fields,- "points", 12345, "", {"+"}, {"-"}).get();+ collection1 = collectionManager.create_collection(schema).get();}virtual void SetUp() {@@ -91,16 +99,135 @@ TEST_F(CollectionManagerTest, CollectionCreation) {ASSERT_EQ(3, num_keys);// we already call `collection1->get_next_seq_id` above, which is side-effectingASSERT_EQ(1, StringUtils::deserialize_uint32_t(next_seq_id));- ASSERT_EQ("{\"created_at\":12345,\"default_sorting_field\":\"points\",\"enable_nested_fields\":false,\"fallback_field_type\":\"\","- "\"fields\":[{\"facet\":false,\"index\":true,\"infix\":false,\"locale\":\"en\",\"name\":\"title\",\"optional\":false,\"sort\":false,\"type\":\"string\"},"- "{\"facet\":false,\"index\":true,\"infix\":true,\"locale\":\"\",\"name\":\"starring\",\"optional\":false,\"sort\":false,\"type\":\"string\"},"- "{\"facet\":true,\"index\":true,\"infix\":false,\"locale\":\"\",\"name\":\"cast\",\"optional\":true,\"sort\":false,\"type\":\"string[]\"},"- "{\"facet\":true,\"index\":true,\"infix\":false,\"locale\":\"\",\"name\":\".*_year\",\"optional\":true,\"sort\":true,\"type\":\"int32\"},"- "{\"facet\":false,\"index\":true,\"infix\":false,\"locale\":\"\",\"name\":\"location\",\"optional\":true,\"sort\":true,\"type\":\"geopoint\"},"- "{\"facet\":false,\"index\":false,\"infix\":false,\"locale\":\"\",\"name\":\"not_stored\",\"optional\":true,\"sort\":false,\"type\":\"string\"},"- "{\"facet\":false,\"index\":true,\"infix\":false,\"locale\":\"\",\"name\":\"points\",\"optional\":false,\"sort\":true,\"type\":\"int32\"}],\"id\":0,"- "\"name\":\"collection1\",\"num_memory_shards\":4,\"symbols_to_index\":[\"+\"],\"token_separators\":[\"-\"]}",- collection_meta_json);++ LOG(INFO) << collection_meta_json;++ nlohmann::json expected_meta_json = R"(+ {+ "created_at":1663234047,+ "default_sorting_field":"points",+ "enable_nested_fields":true,+ "fallback_field_type":"",+ "fields":[+ {+ "facet":false,+ "index":true,+ "infix":false,+ "locale":"en",+ "name":"title",+ "nested":false,+ "optional":false,+ "sort":false,+ "type":"string"+ },+ {+ "facet":false,+ "index":true,+ "infix":true,+ "locale":"",+ "name":"starring",+ "nested":false,+ "optional":false,+ "sort":false,+ "type":"string"+ },+ {+ "facet":true,+ "index":true,+ "infix":false,+ "locale":"",+ "name":"cast",+ "nested":false,+ "optional":true,+ "sort":false,+ "type":"string[]"+ },+ {+ "facet":true,+ "index":true,+ "infix":false,+ "locale":"",+ "name":".*_year",+ "nested":false,+ "optional":true,+ "sort":true,+ "type":"int32"+ },+ {+ "facet":false,+ "index":true,+ "infix":false,+ "locale":"",+ "name":"location",+ "nested":false,+ "optional":true,+ "sort":true,+ "type":"geopoint"+ },+ {+ "facet":false,+ "index":false,+ "infix":false,+ "locale":"",+ "name":"not_stored",+ "nested":false,+ "optional":true,+ "sort":false,+ "type":"string"+ },+ {+ "facet":false,+ "index":true,+ "infix":false,+ "locale":"",+ "name":"points",+ "nested":false,+ "optional":false,+ "sort":true,+ "type":"int32"+ },+ {+ "facet":false,+ "index":true,+ "infix":false,+ "locale":"",+ "name":"person",+ "nested":true,+ "nested_array":2,+ "optional":true,+ "sort":false,+ "type":"object"+ },+ {+ "facet":false,+ "index":true,+ "infix":false,+ "locale":"",+ "name":"vec",+ "nested":false,+ "num_dim":128,+ "optional":true,+ "sort":false,+ "type":"float[]",+ "vec_dist":"cosine"+ }+ ],+ "id":0,+ "name":"collection1",+ "num_memory_shards":4,+ "symbols_to_index":[+ "+"+ ],+ "token_separators":[+ "-"+ ]+ }+ )"_json;++ auto actual_json = nlohmann::json::parse(collection_meta_json);+ expected_meta_json["created_at"] = actual_json["created_at"];++ ASSERT_EQ(expected_meta_json.dump(), actual_json.dump());ASSERT_EQ("1", next_collection_id);}@@ -185,7 +312,15 @@ TEST_F(CollectionManagerTest, GetAllCollections) {ASSERT_STREQ("collection1", collection_vec[0]->get_name().c_str());// try creating one more collection- collectionManager.create_collection("collection2", 4, search_fields, "points");+ auto new_schema = R"({+ "name": "collection2",+ "fields": [+ {"name": "title", "type": "string", "locale": "en"},+ {"name": "points", "type": "int32"}+ ]+ })"_json;++ collectionManager.create_collection(new_schema);collection_vec = collectionManager.get_collections();ASSERT_EQ(2, collection_vec.size());@@ -323,6 +458,11 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {ASSERT_EQ(false, restored_schema.at("title").facet);ASSERT_EQ(false, restored_schema.at("title").optional);ASSERT_EQ(false, restored_schema.at("not_stored").index);+ ASSERT_TRUE(restored_schema.at("person").nested);+ ASSERT_EQ(2, restored_schema.at("person").nested_array);+ ASSERT_EQ(128, restored_schema.at("vec").num_dim);++ ASSERT_TRUE(collection1->get_enable_nested_fields());ASSERT_EQ(2, collection1->get_overrides().size());ASSERT_STREQ("exclude-rule", collection1->get_overrides()["exclude-rule"].id.c_str());