Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Persist vector field properties and restore.

Kishore Nallan49268d648e7d
diff --git a/include/field.h b/include/field.h
index 5ceaf909..0ff3d7d4 100644
--- a/include/field.h
+++ b/include/field.h
@@ -293,6 +293,16 @@ struct field {
field_val[fields::locale] = field.locale;
+ field_val[fields::nested] = field.nested;
+ if(field.nested) {
+ field_val[fields::nested_array] = field.nested_array;
+ }
+
+ if(field.num_dim > 0) {
+ field_val[fields::num_dim] = field.num_dim;
+ field_val[fields::vec_dist] = field.vec_dist == ip ? "ip" : "cosine";
+ }
+
fields_json.push_back(field_val);
if(!field.has_valid_type()) {
diff --git a/src/collection.cpp b/src/collection.cpp
index db0afa35..8ad5e553 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -168,6 +168,15 @@ nlohmann::json Collection::get_summary_json() const {
field_json[fields::sort] = coll_field.sort;
field_json[fields::infix] = coll_field.infix;
field_json[fields::locale] = coll_field.locale;
+ field_json[fields::nested] = coll_field.nested;
+
+ if(coll_field.nested) {
+ field_json[fields::nested_array] = coll_field.nested_array;
+ }
+
+ if(coll_field.num_dim > 0) {
+ field_json[fields::num_dim] = coll_field.num_dim;
+ }
fields_arr.push_back(field_json);
}
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index a85873b4..711ee406 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -42,9 +42,31 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collection
field_obj[fields::infix] = -1;
}
+ if(field_obj.count(fields::nested) == 0) {
+ field_obj[fields::nested] = false;
+ }
+
+ if(field_obj.count(fields::nested_array) == 0) {
+ field_obj[fields::nested_array] = 0;
+ }
+
+ if(field_obj.count(fields::num_dim) == 0) {
+ field_obj[fields::num_dim] = 0;
+ }
+
+ vector_distance_type_t vec_dist_type = vector_distance_type_t::cosine;
+
+ if(field_obj.count(fields::vec_dist) != 0) {
+ auto vec_dist_type_op = magic_enum::enum_cast<vector_distance_type_t>(fields::vec_dist);
+ if(vec_dist_type_op.has_value()) {
+ vec_dist_type = vec_dist_type_op.value();
+ }
+ }
+
field f(field_obj[fields::name], field_obj[fields::type], field_obj[fields::facet],
field_obj[fields::optional], field_obj[fields::index], field_obj[fields::locale],
- -1, field_obj[fields::infix]);
+ -1, field_obj[fields::infix], field_obj[fields::nested], field_obj[fields::nested_array],
+ field_obj[fields::num_dim], vec_dist_type);
// value of `sort` depends on field type
if(field_obj.count(fields::sort) == 0) {
diff --git a/src/index.cpp b/src/index.cpp
index 355e2821..a10f1840 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -2576,6 +2576,8 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
scores[0] = -float_to_int64_t(dist_label.first);
int64_t match_score_index = -1;
+ //LOG(INFO) << "SEQ_ID: " << seq_id << ", score: " << dist_label.first;
+
KV kv(0, searched_queries.size(), 0, seq_id, distinct_id, match_score_index, scores);
topster->add(&kv);
nearest_ids.push_back(seq_id);
diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp
index 61d1f71d..a7fe2f07 100644
--- a/test/collection_manager_test.cpp
+++ b/test/collection_manager_test.cpp
@@ -12,8 +12,8 @@ protected:
CollectionManager & collectionManager = CollectionManager::get_instance();
std::atomic<bool> quit = false;
Collection *collection1;
- std::vector<field> search_fields;
std::vector<sort_by> sort_fields;
+ nlohmann::json schema;
void setupCollection() {
std::string state_dir_path = "/tmp/typesense_test/coll_manager_test_db";
@@ -24,19 +24,27 @@ protected:
collectionManager.init(store, 1.0, "auth_key", quit);
collectionManager.load(8, 1000);
- search_fields = {
- field("title", field_types::STRING, false, false, true, "en", false),
- field("starring", field_types::STRING, false, false, true, "", false, true),
- field("cast", field_types::STRING_ARRAY, true, true, true, "", false),
- field(".*_year", field_types::INT32, true, true),
- field("location", field_types::GEOPOINT, false, true, true),
- field("not_stored", field_types::STRING, false, true, false),
- field("points", field_types::INT32, false)
- };
+ schema = R"({
+ "name": "collection1",
+ "enable_nested_fields": true,
+ "fields": [
+ {"name": "title", "type": "string", "locale": "en"},
+ {"name": "starring", "type": "string", "infix": true},
+ {"name": "cast", "type": "string[]", "facet": true, "optional": true},
+ {"name": ".*_year", "type": "int32", "facet": true, "optional": true},
+ {"name": "location", "type": "geopoint", "optional": true},
+ {"name": "not_stored", "type": "string", "optional": true, "index": false},
+ {"name": "points", "type": "int32"},
+ {"name": "person", "type": "object", "optional": true},
+ {"name": "vec", "type": "float[]", "num_dim": 128, "optional": true}
+ ],
+ "default_sorting_field": "points",
+ "symbols_to_index":["+"],
+ "token_separators":["-"]
+ })"_json;
sort_fields = { sort_by("points", "DESC") };
- collection1 = collectionManager.create_collection("collection1", 4, search_fields,
- "points", 12345, "", {"+"}, {"-"}).get();
+ collection1 = collectionManager.create_collection(schema).get();
}
virtual void SetUp() {
@@ -91,16 +99,135 @@ TEST_F(CollectionManagerTest, CollectionCreation) {
ASSERT_EQ(3, num_keys);
// we already call `collection1->get_next_seq_id` above, which is side-effecting
ASSERT_EQ(1, StringUtils::deserialize_uint32_t(next_seq_id));
- ASSERT_EQ("{\"created_at\":12345,\"default_sorting_field\":\"points\",\"enable_nested_fields\":false,\"fallback_field_type\":\"\","
- "\"fields\":[{\"facet\":false,\"index\":true,\"infix\":false,\"locale\":\"en\",\"name\":\"title\",\"optional\":false,\"sort\":false,\"type\":\"string\"},"
- "{\"facet\":false,\"index\":true,\"infix\":true,\"locale\":\"\",\"name\":\"starring\",\"optional\":false,\"sort\":false,\"type\":\"string\"},"
- "{\"facet\":true,\"index\":true,\"infix\":false,\"locale\":\"\",\"name\":\"cast\",\"optional\":true,\"sort\":false,\"type\":\"string[]\"},"
- "{\"facet\":true,\"index\":true,\"infix\":false,\"locale\":\"\",\"name\":\".*_year\",\"optional\":true,\"sort\":true,\"type\":\"int32\"},"
- "{\"facet\":false,\"index\":true,\"infix\":false,\"locale\":\"\",\"name\":\"location\",\"optional\":true,\"sort\":true,\"type\":\"geopoint\"},"
- "{\"facet\":false,\"index\":false,\"infix\":false,\"locale\":\"\",\"name\":\"not_stored\",\"optional\":true,\"sort\":false,\"type\":\"string\"},"
- "{\"facet\":false,\"index\":true,\"infix\":false,\"locale\":\"\",\"name\":\"points\",\"optional\":false,\"sort\":true,\"type\":\"int32\"}],\"id\":0,"
- "\"name\":\"collection1\",\"num_memory_shards\":4,\"symbols_to_index\":[\"+\"],\"token_separators\":[\"-\"]}",
- collection_meta_json);
+
+ LOG(INFO) << collection_meta_json;
+
+ nlohmann::json expected_meta_json = R"(
+ {
+ "created_at":1663234047,
+ "default_sorting_field":"points",
+ "enable_nested_fields":true,
+ "fallback_field_type":"",
+ "fields":[
+ {
+ "facet":false,
+ "index":true,
+ "infix":false,
+ "locale":"en",
+ "name":"title",
+ "nested":false,
+ "optional":false,
+ "sort":false,
+ "type":"string"
+ },
+ {
+ "facet":false,
+ "index":true,
+ "infix":true,
+ "locale":"",
+ "name":"starring",
+ "nested":false,
+ "optional":false,
+ "sort":false,
+ "type":"string"
+ },
+ {
+ "facet":true,
+ "index":true,
+ "infix":false,
+ "locale":"",
+ "name":"cast",
+ "nested":false,
+ "optional":true,
+ "sort":false,
+ "type":"string[]"
+ },
+ {
+ "facet":true,
+ "index":true,
+ "infix":false,
+ "locale":"",
+ "name":".*_year",
+ "nested":false,
+ "optional":true,
+ "sort":true,
+ "type":"int32"
+ },
+ {
+ "facet":false,
+ "index":true,
+ "infix":false,
+ "locale":"",
+ "name":"location",
+ "nested":false,
+ "optional":true,
+ "sort":true,
+ "type":"geopoint"
+ },
+ {
+ "facet":false,
+ "index":false,
+ "infix":false,
+ "locale":"",
+ "name":"not_stored",
+ "nested":false,
+ "optional":true,
+ "sort":false,
+ "type":"string"
+ },
+ {
+ "facet":false,
+ "index":true,
+ "infix":false,
+ "locale":"",
+ "name":"points",
+ "nested":false,
+ "optional":false,
+ "sort":true,
+ "type":"int32"
+ },
+ {
+ "facet":false,
+ "index":true,
+ "infix":false,
+ "locale":"",
+ "name":"person",
+ "nested":true,
+ "nested_array":2,
+ "optional":true,
+ "sort":false,
+ "type":"object"
+ },
+ {
+ "facet":false,
+ "index":true,
+ "infix":false,
+ "locale":"",
+ "name":"vec",
+ "nested":false,
+ "num_dim":128,
+ "optional":true,
+ "sort":false,
+ "type":"float[]",
+ "vec_dist":"cosine"
+ }
+ ],
+ "id":0,
+ "name":"collection1",
+ "num_memory_shards":4,
+ "symbols_to_index":[
+ "+"
+ ],
+ "token_separators":[
+ "-"
+ ]
+ }
+ )"_json;
+
+ auto actual_json = nlohmann::json::parse(collection_meta_json);
+ expected_meta_json["created_at"] = actual_json["created_at"];
+
+ ASSERT_EQ(expected_meta_json.dump(), actual_json.dump());
ASSERT_EQ("1", next_collection_id);
}
@@ -185,7 +312,15 @@ TEST_F(CollectionManagerTest, GetAllCollections) {
ASSERT_STREQ("collection1", collection_vec[0]->get_name().c_str());
// try creating one more collection
- collectionManager.create_collection("collection2", 4, search_fields, "points");
+ auto new_schema = R"({
+ "name": "collection2",
+ "fields": [
+ {"name": "title", "type": "string", "locale": "en"},
+ {"name": "points", "type": "int32"}
+ ]
+ })"_json;
+
+ collectionManager.create_collection(new_schema);
collection_vec = collectionManager.get_collections();
ASSERT_EQ(2, collection_vec.size());
@@ -323,6 +458,11 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {
ASSERT_EQ(false, restored_schema.at("title").facet);
ASSERT_EQ(false, restored_schema.at("title").optional);
ASSERT_EQ(false, restored_schema.at("not_stored").index);
+ ASSERT_TRUE(restored_schema.at("person").nested);
+ ASSERT_EQ(2, restored_schema.at("person").nested_array);
+ ASSERT_EQ(128, restored_schema.at("vec").num_dim);
+
+ ASSERT_TRUE(collection1->get_enable_nested_fields());
ASSERT_EQ(2, collection1->get_overrides().size());
ASSERT_STREQ("exclude-rule", collection1->get_overrides()["exclude-rule"].id.c_str());