Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Merge branch 'v0.25-join' into v0.26-facets

Kishore Nallana713c2675fbd
diff --cc src/field.cpp
index fec95589,d053e578..9ef6e643
--- a/src/field.cpp
+++ b/src/field.cpp
@@@ -625,45 -1108,207 +621,207 @@@ Option<bool> field::json_fields_to_fiel
return Option<bool>(true);
}
- Option<bool> field::validate_and_init_embed_fields(const std::vector<std::pair<size_t, size_t>>& embed_json_field_indices,
- const tsl::htrie_map<char, field>& search_schema,
- nlohmann::json& fields_json,
- std::vector<field>& fields_vec) {
-
- for(const auto& json_field_index: embed_json_field_indices) {
- auto& field_json = fields_json[json_field_index.first];
- const std::string err_msg = "Property `" + fields::embed + "." + fields::from +
+ Option<bool> field::validate_and_init_embed_field(const tsl::htrie_map<char, field>& search_schema, nlohmann::json& field_json,
+ const nlohmann::json& fields_json,
+ field& the_field) {
+ const std::string err_msg = "Property `" + fields::embed + "." + fields::from +
"` can only refer to string or string array fields.";
- for(auto& field_name : field_json[fields::embed][fields::from].get<std::vector<std::string>>()) {
- auto embed_field = std::find_if(fields_json.begin(), fields_json.end(), [&field_name](const nlohmann::json& x) {
- return x["name"].get<std::string>() == field_name;
- });
-
- if(embed_field == fields_json.end()) {
- const auto& embed_field2 = search_schema.find(field_name);
- if (embed_field2 == search_schema.end()) {
- return Option<bool>(400, err_msg);
- } else if (embed_field2->type != field_types::STRING && embed_field2->type != field_types::STRING_ARRAY) {
- return Option<bool>(400, err_msg);
- }
- } else if((*embed_field)[fields::type] != field_types::STRING &&
- (*embed_field)[fields::type] != field_types::STRING_ARRAY) {
+ for(auto& field_name : field_json[fields::embed][fields::from].get<std::vector<std::string>>()) {
+
+ auto embed_field = std::find_if(fields_json.begin(), fields_json.end(), [&field_name](const nlohmann::json& x) {
+ return x["name"].get<std::string>() == field_name;
+ });
+
+
+ if(embed_field == fields_json.end()) {
+ const auto& embed_field2 = search_schema.find(field_name);
+ if (embed_field2 == search_schema.end()) {
return Option<bool>(400, err_msg);
+ } else if (embed_field2->type != field_types::STRING && embed_field2->type != field_types::STRING_ARRAY) {
+ return Option<bool>(400, err_msg);
+ }
+ } else if((*embed_field)[fields::type] != field_types::STRING &&
+ (*embed_field)[fields::type] != field_types::STRING_ARRAY) {
+ return Option<bool>(400, err_msg);
+ }
+ }
+
+ const auto& model_config = field_json[fields::embed][fields::model_config];
+ size_t num_dim = 0;
+ auto res = TextEmbedderManager::get_instance().validate_and_init_model(model_config, num_dim);
+ if(!res.ok()) {
+ return Option<bool>(res.code(), res.error());
+ }
+
+ LOG(INFO) << "Model init done.";
+ field_json[fields::num_dim] = num_dim;
+ the_field.num_dim = num_dim;
+
+ return Option<bool>(true);
+ }
+
-
+ void filter_result_t::and_filter_results(const filter_result_t& a, const filter_result_t& b, filter_result_t& result) {
+ auto lenA = a.count, lenB = b.count;
+ if (lenA == 0 || lenB == 0) {
+ return;
+ }
+
+ result.docs = new uint32_t[std::min(lenA, lenB)];
+
+ auto A = a.docs, B = b.docs, out = result.docs;
+ const uint32_t *endA = A + lenA;
+ const uint32_t *endB = B + lenB;
+
+ // Add an entry of references in the result for each unique collection in a and b.
+ for (auto const& item: a.reference_filter_results) {
+ if (result.reference_filter_results.count(item.first) == 0) {
+ result.reference_filter_results[item.first] = new reference_filter_result_t[std::min(lenA, lenB)];
+ }
+ }
+ for (auto const& item: b.reference_filter_results) {
+ if (result.reference_filter_results.count(item.first) == 0) {
+ result.reference_filter_results[item.first] = new reference_filter_result_t[std::min(lenA, lenB)];
+ }
+ }
+
+ while (true) {
+ while (*A < *B) {
+ SKIP_FIRST_COMPARE:
+ if (++A == endA) {
+ result.count = out - result.docs;
+ return;
+ }
+ }
+ while (*A > *B) {
+ if (++B == endB) {
+ result.count = out - result.docs;
+ return;
}
}
+ if (*A == *B) {
+ *out = *A;
+
+ // Copy the references of the document from every collection into result.
+ for (auto const& item: a.reference_filter_results) {
+ result.reference_filter_results[item.first][out - result.docs] = item.second[A - a.docs];
+ }
+ for (auto const& item: b.reference_filter_results) {
+ result.reference_filter_results[item.first][out - result.docs] = item.second[B - b.docs];
+ }
+
+ out++;
- const auto& model_config = field_json[fields::embed][fields::model_config];
- size_t num_dim = 0;
- auto res = TextEmbedderManager::get_instance().validate_and_init_model(model_config, num_dim);
- if(!res.ok()) {
- return Option<bool>(res.code(), res.error());
+ if (++A == endA || ++B == endB) {
+ result.count = out - result.docs;
+ return;
+ }
+ } else {
+ goto SKIP_FIRST_COMPARE;
}
+ }
+ }
- LOG(INFO) << "Model init done.";
- field_json[fields::num_dim] = num_dim;
- fields_vec[json_field_index.second].num_dim = num_dim;
+ void filter_result_t::or_filter_results(const filter_result_t& a, const filter_result_t& b, filter_result_t& result) {
+ if (a.count == 0 && b.count == 0) {
+ return;
}
- return Option<bool>(true);
+ // If either one of a or b does not have any matches, copy other into result.
+ if (a.count == 0) {
+ result = b;
+ return;
+ }
+ if (b.count == 0) {
+ result = a;
+ return;
+ }
+
+ size_t indexA = 0, indexB = 0, res_index = 0, lenA = a.count, lenB = b.count;
+ result.docs = new uint32_t[lenA + lenB];
+
+ // Add an entry of references in the result for each unique collection in a and b.
+ for (auto const& item: a.reference_filter_results) {
+ if (result.reference_filter_results.count(item.first) == 0) {
+ result.reference_filter_results[item.first] = new reference_filter_result_t[lenA + lenB];
+ }
+ }
+ for (auto const& item: b.reference_filter_results) {
+ if (result.reference_filter_results.count(item.first) == 0) {
+ result.reference_filter_results[item.first] = new reference_filter_result_t[lenA + lenB];
+ }
+ }
+
+ while (indexA < lenA && indexB < lenB) {
+ if (a.docs[indexA] < b.docs[indexB]) {
+ // check for duplicate
+ if (res_index == 0 || result.docs[res_index - 1] != a.docs[indexA]) {
+ result.docs[res_index] = a.docs[indexA];
+ res_index++;
+ }
+
+ // Copy references of the last result document from every collection in a.
+ for (auto const& item: a.reference_filter_results) {
+ result.reference_filter_results[item.first][res_index - 1] = item.second[indexA];
+ }
+
+ indexA++;
+ } else {
+ if (res_index == 0 || result.docs[res_index - 1] != b.docs[indexB]) {
+ result.docs[res_index] = b.docs[indexB];
+ res_index++;
+ }
+
+ for (auto const& item: b.reference_filter_results) {
+ result.reference_filter_results[item.first][res_index - 1] = item.second[indexB];
+ }
+
+ indexB++;
+ }
+ }
+
+ while (indexA < lenA) {
+ if (res_index == 0 || result.docs[res_index - 1] != a.docs[indexA]) {
+ result.docs[res_index] = a.docs[indexA];
+ res_index++;
+ }
+
+ for (auto const& item: a.reference_filter_results) {
+ result.reference_filter_results[item.first][res_index - 1] = item.second[indexA];
+ }
+
+ indexA++;
+ }
+
+ while (indexB < lenB) {
+ if(res_index == 0 || result.docs[res_index - 1] != b.docs[indexB]) {
+ result.docs[res_index] = b.docs[indexB];
+ res_index++;
+ }
+
+ for (auto const& item: b.reference_filter_results) {
+ result.reference_filter_results[item.first][res_index - 1] = item.second[indexB];
+ }
+
+ indexB++;
+ }
+
+ result.count = res_index;
+
+ // shrink fit
+ auto out = new uint32_t[res_index];
+ memcpy(out, result.docs, res_index * sizeof(uint32_t));
+ delete[] result.docs;
+ result.docs = out;
+
+ for (auto &item: result.reference_filter_results) {
+ auto out_references = new reference_filter_result_t[res_index];
+
+ for (uint32_t i = 0; i < result.count; i++) {
+ out_references[i] = item.second[i];
+ }
+ delete[] item.second;
+ item.second = out_references;
+ }
}
++