Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Add validation for vector dimensions during indexing.

Kishore Nallanddfb7b27bb70
diff --git a/src/validator.cpp b/src/validator.cpp
index bf6ba641..25a77884 100644
--- a/src/validator.cpp
+++ b/src/validator.cpp
@@ -77,6 +77,11 @@ Option<uint32_t> validator_t::coerce_element(const field& a_field, nlohmann::jso
return Option<uint32_t>(200);
}
+ if(a_field.type == field_types::FLOAT_ARRAY && a_field.num_dim != 0 && a_field.num_dim != doc_ele.size()) {
+ return Option<uint32_t>(400, "Field `" + a_field.name + "` must have " +
+ std::to_string(a_field.num_dim) + " dimensions.");
+ }
+
for(; it != doc_ele.end(); ) {
const auto& item = it.value();
array_ele_erased = false;
diff --git a/test/collection_vector_search_test.cpp b/test/collection_vector_search_test.cpp
index 2e207bbd..36381780 100644
--- a/test/collection_vector_search_test.cpp
+++ b/test/collection_vector_search_test.cpp
@@ -3,6 +3,8 @@
#include <vector>
#include <collection_manager.h>
#include "collection.h"
+#include <cstdlib>
+#include <ctime>
class CollectionVectorTest : public ::testing::Test {
protected:
@@ -223,6 +225,38 @@ TEST_F(CollectionVectorTest, BasicVectorQuerying) {
collectionManager.drop_collection("coll1");
}
+TEST_F(CollectionVectorTest, NumVectorGreaterThanNumDim) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "fields": [
+ {"name": "title", "type": "string"},
+ {"name": "points", "type": "int32"},
+ {"name": "vec", "type": "float[]", "num_dim": 3}
+ ]
+ })"_json;
+
+ Collection* coll1 = collectionManager.create_collection(schema).get();
+
+ srand (static_cast <unsigned> (time(0)));
+
+ for(size_t i = 0; i < 10; i++) {
+ nlohmann::json doc;
+ doc["id"] = std::to_string(i);
+ doc["title"] = "Title";
+ doc["points"] = 100;
+ doc["vec"] = std::vector<float>();
+
+ for(size_t j = 0; j < 100; j++) {
+ float r = static_cast <float> (rand()) / static_cast <float> (RAND_MAX);
+ doc["vec"].push_back(r);
+ }
+
+ auto add_op = coll1->add(doc.dump());
+ ASSERT_FALSE(add_op.ok());
+ ASSERT_EQ("Field `vec` must have 3 dimensions.", add_op.error());
+ }
+}
+
TEST_F(CollectionVectorTest, IndexGreaterThan1KVectors) {
// tests the dynamic resizing of graph
nlohmann::json schema = R"({