Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Allow filtering and deleting using doc IDs.
Kishore Nallan5b0690fcd899
diff --git a/src/collection.cpp b/src/collection.cppindex 036d7680..f277c5d3 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -2104,6 +2104,44 @@ Option<bool> Collection::parse_filter_query(const std::string& simple_filter_questd::string&& field_name = filter_block.substr(0, found_index);StringUtils::trim(field_name);+ if(field_name == "id") {+ filter id_filter;+ std::string&& raw_value = filter_block.substr(found_index+1, std::string::npos);+ StringUtils::trim(raw_value);+ id_filter = {field_name, {}, {}};++ if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {+ std::vector<std::string> doc_ids;+ StringUtils::split(raw_value.substr(1, raw_value.size() - 2), doc_ids, ",");++ for(std::string& doc_id: doc_ids) {+ // we have to convert the doc_id to seq id+ std::string seq_id_str;+ StoreStatus seq_id_status = store->get(get_doc_id_key(doc_id), seq_id_str);++ if(seq_id_status != StoreStatus::FOUND) {+ continue;+ }++ id_filter.values.push_back(seq_id_str);+ id_filter.comparators.push_back(EQUALS);+ }+ } else {+ std::string seq_id_str;+ StoreStatus seq_id_status = store->get(get_doc_id_key(raw_value), seq_id_str);+ if(seq_id_status == StoreStatus::FOUND) {+ id_filter.values.push_back(seq_id_str);+ id_filter.comparators.push_back(EQUALS);+ }+ }++ if(!id_filter.values.empty()) {+ filters.push_back(id_filter);+ }++ continue;+ }+if(search_schema.count(field_name) == 0) {return Option<bool>(404, "Could not find a filter field named `" + field_name + "` in the schema.");}diff --git a/src/index.cpp b/src/index.cppindex a98453b7..c61f9981 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1080,6 +1080,29 @@ uint32_t Index::do_filtering(uint32_t** filter_ids_out, const std::vector<filterfor(size_t i = 0; i < filters.size(); i++) {const filter & a_filter = filters[i];++ if(a_filter.field_name == "id") {+ // we handle `ids` separately+ std::vector<uint32> result_ids;+ for(const auto& id_str: a_filter.values) {+ result_ids.push_back(std::stoul(id_str));+ }++ if(i == 0) {+ filter_ids = new uint32[result_ids.size()];+ std::copy(result_ids.begin(), result_ids.end(), filter_ids);+ filter_ids_length = result_ids.size();+ } else {+ uint32_t* filtered_results = nullptr;+ filter_ids_length = ArrayUtils::and_scalar(filter_ids, filter_ids_length, &result_ids[0],+ result_ids.size(), &filtered_results);+ delete [] filter_ids;+ filter_ids = filtered_results;+ }++ continue;+ }+bool has_search_index = search_index.count(a_filter.field_name) != 0 ||numerical_index.count(a_filter.field_name) != 0 ||geopoint_index.count(a_filter.field_name) != 0;diff --git a/test/collection_filtering_test.cpp b/test/collection_filtering_test.cppindex e70612bb..51b31796 100644--- a/test/collection_filtering_test.cpp+++ b/test/collection_filtering_test.cpp@@ -1311,6 +1311,67 @@ TEST_F(CollectionFilteringTest, NumericalFilteringWithAnd) {collectionManager.drop_collection("coll1");}+TEST_F(CollectionFilteringTest, FilteringViaDocumentIds) {+ Collection *coll1;++ std::vector<field> fields = {field("company_name", field_types::STRING, false),+ field("num_employees", field_types::INT32, false),};++ coll1 = collectionManager.get_collection("coll1").get();+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "num_employees").get();+ }++ std::vector<std::vector<std::string>> records = {+ {"123", "Company 1", "50"},+ {"125", "Company 2", "150"},+ {"127", "Company 3", "250"},+ {"129", "Stark Industries 4", "500"},+ };++ for(size_t i=0; i<records.size(); i++) {+ nlohmann::json doc;++ doc["id"] = records[i][0];+ doc["company_name"] = records[i][1];+ doc["num_employees"] = std::stoi(records[i][2]);++ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ std::vector<sort_by> sort_fields = { sort_by("num_employees", "ASC") };++ auto results = coll1->search("*",+ {}, "id: 123",+ {}, sort_fields, {0}, 10, 1, FREQUENCY, {true}).get();++ ASSERT_EQ(1, results["found"].get<size_t>());+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_STREQ("123", results["hits"][0]["document"]["id"].get<std::string>().c_str());++ // single ID with condition+ results = coll1->search("*",+ {}, "id: 125 && num_employees: 150",+ {}, sort_fields, {0}, 10, 1, FREQUENCY, {true}).get();++ ASSERT_EQ(1, results["found"].get<size_t>());+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_STREQ("125", results["hits"][0]["document"]["id"].get<std::string>().c_str());++ // multiple IDs+ results = coll1->search("*",+ {}, "id: [123, 125, 127, 129] && num_employees: <300",+ {}, sort_fields, {0}, 10, 1, FREQUENCY, {true}).get();++ ASSERT_EQ(3, results["found"].get<size_t>());+ ASSERT_EQ(3, results["hits"].size());+ ASSERT_STREQ("123", results["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("125", results["hits"][1]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("127", results["hits"][2]["document"]["id"].get<std::string>().c_str());++ collectionManager.drop_collection("coll1");+}+TEST_F(CollectionFilteringTest, NumericalFilteringWithArray) {Collection *coll1;diff --git a/test/core_api_utils_test.cpp b/test/core_api_utils_test.cppindex 201d76b5..0492bb6a 100644--- a/test/core_api_utils_test.cpp+++ b/test/core_api_utils_test.cpp@@ -111,6 +111,45 @@ TEST_F(CoreAPIUtilsTest, StatefulRemoveDocs) {ASSERT_EQ(9, deletion_state.num_removed);ASSERT_TRUE(done);+ // fetch raw document IDs+ for(size_t i=0; i<100; i++) {+ nlohmann::json doc;++ doc["id"] = std::to_string(i);+ doc["title"] = "Title " + std::to_string(i);+ doc["points"] = i;++ coll1->add(doc.dump());+ }++ deletion_state.index_ids.clear();+ deletion_state.offsets.clear();+ deletion_state.num_removed = 0;++ coll1->get_filter_ids("id:[0, 1, 2]", deletion_state.index_ids);+ for(size_t i=0; i<deletion_state.index_ids.size(); i++) {+ deletion_state.offsets.push_back(0);+ }++ stateful_remove_docs(&deletion_state, 5, done);+ ASSERT_EQ(3, deletion_state.num_removed);+ ASSERT_TRUE(done);++ // delete single doc++ deletion_state.index_ids.clear();+ deletion_state.offsets.clear();+ deletion_state.num_removed = 0;++ coll1->get_filter_ids("id: 10", deletion_state.index_ids);+ for(size_t i=0; i<deletion_state.index_ids.size(); i++) {+ deletion_state.offsets.push_back(0);+ }++ stateful_remove_docs(&deletion_state, 5, done);+ ASSERT_EQ(1, deletion_state.num_removed);+ ASSERT_TRUE(done);+// bad filter queryauto op = coll1->get_filter_ids("bad filter", deletion_state.index_ids);ASSERT_FALSE(op.ok());