Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Skip empty strings during indexing.

Kishore Nallan239f4443e804
diff --git a/src/index.cpp b/src/index.cpp
index 86f604bb..2b80f9e9 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -573,6 +573,10 @@ void Index::index_string_field(const std::string & text, const int64_t score, ar
last_token = token;
}
+ if(token_to_offsets.empty()) {
+ return;
+ }
+
// push 0 for the last occurring token (used for exact match ranking)
token_to_offsets[last_token].push_back(0);
@@ -625,6 +629,10 @@ void Index::index_string_array_field(const std::vector<std::string> & strings, c
last_token = token;
}
+ if(token_set.empty()) {
+ continue;
+ }
+
if(is_facet) {
facet_hashes.push_back(FACET_ARRAY_DELIMETER); // as a delimiter
}
@@ -641,6 +649,10 @@ void Index::index_string_array_field(const std::vector<std::string> & strings, c
token_positions[last_token].push_back(0);
}
+ if(token_positions.empty()) {
+ return;
+ }
+
if(is_facet) {
facet_hash_values_t fhashvalues;
fhashvalues.length = facet_hashes.size();
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index df3c16bf..29a069c9 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -2632,6 +2632,48 @@ TEST_F(CollectionTest, OptionalFieldCanBeNull) {
collectionManager.drop_collection("coll1");
}
+TEST_F(CollectionTest, EmptyStringNotIndexed) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("title", field_types::STRING, false),
+ field("artist", field_types::STRING, false, true),
+ field("genres", field_types::STRING_ARRAY, false, true),
+ field("launch_year", field_types::STRING, false, true),
+ field("labels", field_types::STRING_ARRAY, false, true),
+ field("points", field_types::INT32, false),};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
+ }
+
+ nlohmann::json doc;
+
+ doc["id"] = "0";
+ doc["title"] = "Beat it";
+ doc["artist"] = "";
+ doc["launch_year"] = " ";
+ doc["genres"] = {""};
+ doc["labels"] = {"song", " ", ""};
+ doc["points"] = 100;
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ auto results = coll1->search("beat",
+ {"title", "artist"}, "", {}, {}, 0, 10, 1, FREQUENCY).get();
+
+ ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_EQ(1, results["hits"].size());
+
+ ASSERT_EQ(2, coll1->_get_indexes()[0]->_get_search_index().at("title")->size);
+ ASSERT_EQ(0, coll1->_get_indexes()[0]->_get_search_index().at("artist")->size);
+ ASSERT_EQ(0, coll1->_get_indexes()[0]->_get_search_index().at("launch_year")->size);
+ ASSERT_EQ(0, coll1->_get_indexes()[0]->_get_search_index().at("genres")->size);
+ ASSERT_EQ(1, coll1->_get_indexes()[0]->_get_search_index().at("labels")->size);
+
+ collectionManager.drop_collection("coll1");
+}
+
TEST_F(CollectionTest, WildcardQueryReturnsResultsBasedOnPerPageParam) {
std::vector<std::string> facets;
spp::sparse_hash_set<std::string> empty;