Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Allow filtering and deleting using doc IDs.

Kishore Nallan5b0690fcd899
diff --git a/src/collection.cpp b/src/collection.cpp
index 036d7680..f277c5d3 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -2104,6 +2104,44 @@ Option<bool> Collection::parse_filter_query(const std::string& simple_filter_que
std::string&& field_name = filter_block.substr(0, found_index);
StringUtils::trim(field_name);
+ if(field_name == "id") {
+ filter id_filter;
+ std::string&& raw_value = filter_block.substr(found_index+1, std::string::npos);
+ StringUtils::trim(raw_value);
+ id_filter = {field_name, {}, {}};
+
+ if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
+ std::vector<std::string> doc_ids;
+ StringUtils::split(raw_value.substr(1, raw_value.size() - 2), doc_ids, ",");
+
+ for(std::string& doc_id: doc_ids) {
+ // we have to convert the doc_id to seq id
+ std::string seq_id_str;
+ StoreStatus seq_id_status = store->get(get_doc_id_key(doc_id), seq_id_str);
+
+ if(seq_id_status != StoreStatus::FOUND) {
+ continue;
+ }
+
+ id_filter.values.push_back(seq_id_str);
+ id_filter.comparators.push_back(EQUALS);
+ }
+ } else {
+ std::string seq_id_str;
+ StoreStatus seq_id_status = store->get(get_doc_id_key(raw_value), seq_id_str);
+ if(seq_id_status == StoreStatus::FOUND) {
+ id_filter.values.push_back(seq_id_str);
+ id_filter.comparators.push_back(EQUALS);
+ }
+ }
+
+ if(!id_filter.values.empty()) {
+ filters.push_back(id_filter);
+ }
+
+ continue;
+ }
+
if(search_schema.count(field_name) == 0) {
return Option<bool>(404, "Could not find a filter field named `" + field_name + "` in the schema.");
}
diff --git a/src/index.cpp b/src/index.cpp
index a98453b7..c61f9981 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1080,6 +1080,29 @@ uint32_t Index::do_filtering(uint32_t** filter_ids_out, const std::vector<filter
for(size_t i = 0; i < filters.size(); i++) {
const filter & a_filter = filters[i];
+
+ if(a_filter.field_name == "id") {
+ // we handle `ids` separately
+ std::vector<uint32> result_ids;
+ for(const auto& id_str: a_filter.values) {
+ result_ids.push_back(std::stoul(id_str));
+ }
+
+ if(i == 0) {
+ filter_ids = new uint32[result_ids.size()];
+ std::copy(result_ids.begin(), result_ids.end(), filter_ids);
+ filter_ids_length = result_ids.size();
+ } else {
+ uint32_t* filtered_results = nullptr;
+ filter_ids_length = ArrayUtils::and_scalar(filter_ids, filter_ids_length, &result_ids[0],
+ result_ids.size(), &filtered_results);
+ delete [] filter_ids;
+ filter_ids = filtered_results;
+ }
+
+ continue;
+ }
+
bool has_search_index = search_index.count(a_filter.field_name) != 0 ||
numerical_index.count(a_filter.field_name) != 0 ||
geopoint_index.count(a_filter.field_name) != 0;
diff --git a/test/collection_filtering_test.cpp b/test/collection_filtering_test.cpp
index e70612bb..51b31796 100644
--- a/test/collection_filtering_test.cpp
+++ b/test/collection_filtering_test.cpp
@@ -1311,6 +1311,67 @@ TEST_F(CollectionFilteringTest, NumericalFilteringWithAnd) {
collectionManager.drop_collection("coll1");
}
+TEST_F(CollectionFilteringTest, FilteringViaDocumentIds) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("company_name", field_types::STRING, false),
+ field("num_employees", field_types::INT32, false),};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "num_employees").get();
+ }
+
+ std::vector<std::vector<std::string>> records = {
+ {"123", "Company 1", "50"},
+ {"125", "Company 2", "150"},
+ {"127", "Company 3", "250"},
+ {"129", "Stark Industries 4", "500"},
+ };
+
+ for(size_t i=0; i<records.size(); i++) {
+ nlohmann::json doc;
+
+ doc["id"] = records[i][0];
+ doc["company_name"] = records[i][1];
+ doc["num_employees"] = std::stoi(records[i][2]);
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+ }
+
+ std::vector<sort_by> sort_fields = { sort_by("num_employees", "ASC") };
+
+ auto results = coll1->search("*",
+ {}, "id: 123",
+ {}, sort_fields, {0}, 10, 1, FREQUENCY, {true}).get();
+
+ ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_STREQ("123", results["hits"][0]["document"]["id"].get<std::string>().c_str());
+
+ // single ID with condition
+ results = coll1->search("*",
+ {}, "id: 125 && num_employees: 150",
+ {}, sort_fields, {0}, 10, 1, FREQUENCY, {true}).get();
+
+ ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_STREQ("125", results["hits"][0]["document"]["id"].get<std::string>().c_str());
+
+ // multiple IDs
+ results = coll1->search("*",
+ {}, "id: [123, 125, 127, 129] && num_employees: <300",
+ {}, sort_fields, {0}, 10, 1, FREQUENCY, {true}).get();
+
+ ASSERT_EQ(3, results["found"].get<size_t>());
+ ASSERT_EQ(3, results["hits"].size());
+ ASSERT_STREQ("123", results["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("125", results["hits"][1]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("127", results["hits"][2]["document"]["id"].get<std::string>().c_str());
+
+ collectionManager.drop_collection("coll1");
+}
+
TEST_F(CollectionFilteringTest, NumericalFilteringWithArray) {
Collection *coll1;
diff --git a/test/core_api_utils_test.cpp b/test/core_api_utils_test.cpp
index 201d76b5..0492bb6a 100644
--- a/test/core_api_utils_test.cpp
+++ b/test/core_api_utils_test.cpp
@@ -111,6 +111,45 @@ TEST_F(CoreAPIUtilsTest, StatefulRemoveDocs) {
ASSERT_EQ(9, deletion_state.num_removed);
ASSERT_TRUE(done);
+ // fetch raw document IDs
+ for(size_t i=0; i<100; i++) {
+ nlohmann::json doc;
+
+ doc["id"] = std::to_string(i);
+ doc["title"] = "Title " + std::to_string(i);
+ doc["points"] = i;
+
+ coll1->add(doc.dump());
+ }
+
+ deletion_state.index_ids.clear();
+ deletion_state.offsets.clear();
+ deletion_state.num_removed = 0;
+
+ coll1->get_filter_ids("id:[0, 1, 2]", deletion_state.index_ids);
+ for(size_t i=0; i<deletion_state.index_ids.size(); i++) {
+ deletion_state.offsets.push_back(0);
+ }
+
+ stateful_remove_docs(&deletion_state, 5, done);
+ ASSERT_EQ(3, deletion_state.num_removed);
+ ASSERT_TRUE(done);
+
+ // delete single doc
+
+ deletion_state.index_ids.clear();
+ deletion_state.offsets.clear();
+ deletion_state.num_removed = 0;
+
+ coll1->get_filter_ids("id: 10", deletion_state.index_ids);
+ for(size_t i=0; i<deletion_state.index_ids.size(); i++) {
+ deletion_state.offsets.push_back(0);
+ }
+
+ stateful_remove_docs(&deletion_state, 5, done);
+ ASSERT_EQ(1, deletion_state.num_removed);
+ ASSERT_TRUE(done);
+
// bad filter query
auto op = coll1->get_filter_ids("bad filter", deletion_state.index_ids);
ASSERT_FALSE(op.ok());