Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Auto vector generation & Hybrid Search
ozanarmagan2672b1ebd6e5
diff --git a/WORKSPACE b/WORKSPACEindex 94d24423..1dc1401d 100644--- a/WORKSPACE+++ b/WORKSPACE@@ -13,10 +13,7 @@ bazel_compdb_deps()http_archive(name = "rules_foreign_cc",- patches = ["//bazel:foreign_cc.patch", "//bazel:foreign_cc_version_compiler.patch"],- patch_args = [- "-p1",- ],+ patches = ["//bazel:foreign_cc.patch"],sha256 = "2a4d07cd64b0719b39a7c12218a3e507672b82a97b98c6a89d38565894cf7c51",strip_prefix = "rules_foreign_cc-0.9.0",url = "https://github.com/bazelbuild/rules_foreign_cc/archive/refs/tags/0.9.0.tar.gz",diff --git a/bazel/foreign_cc.patch b/bazel/foreign_cc.patchindex 9cb52c7c..9af0c8a7 100644--- a/bazel/foreign_cc.patch+++ b/bazel/foreign_cc.patch@@ -1,5 +1,5 @@---- a/foreign_cc/private/configure_script.bzl-+++ b/foreign_cc/private/configure_script.bzl+--- foreign_cc/private/configure_script.bzl++++ foreign_cc/private/configure_script.bzl@@ -70,7 +70,7 @@).lstrip())diff --git a/bazel/foreign_cc_version_compiler.patch b/bazel/foreign_cc_version_compiler.patchdeleted file mode 100644index d668a1db..00000000--- a/bazel/foreign_cc_version_compiler.patch+++ /dev/null@@ -1,283 +0,0 @@-diff --git a/foreign_cc/private/cc_toolchain_util.bzl b/foreign_cc/private/cc_toolchain_util.bzl-index fd7fa4d..188dc5f 100644---- a/foreign_cc/private/cc_toolchain_util.bzl-+++ b/foreign_cc/private/cc_toolchain_util.bzl-@@ -265,15 +265,24 @@ def get_tools_info(ctx):- cc_toolchain = cc_toolchain,- )--+ cxx = cc_common.get_tool_for_action(-+ feature_configuration = feature_configuration,-+ action_name = ACTION_NAMES.cpp_compile,-+ )-+ cxx_splitted = cxx.split("/")-+ if(cxx_splitted[-1] == "gcc"):-+ cxx_splitted[-1] = "g++"-+ cxx = "/".join(cxx_splitted)-+ if(cxx_splitted[-1] == "clang"):-+ cxx_splitted = "clang++"-+ cxx = "/".join(cxx_splitted)-+- return CxxToolsInfo(- cc = cc_common.get_tool_for_action(- feature_configuration = feature_configuration,- action_name = ACTION_NAMES.c_compile,- ),-- cxx = cc_common.get_tool_for_action(-- feature_configuration = feature_configuration,-- action_name = ACTION_NAMES.cpp_compile,-- ),-+ cxx = cxx,- cxx_linker_static = cc_common.get_tool_for_action(- feature_configuration = feature_configuration,- action_name = ACTION_NAMES.cpp_link_static_library,-diff --git a/toolchains/built_toolchains.bzl b/toolchains/built_toolchains.bzl-index 5e59e79..ddf63a5 100644---- a/toolchains/built_toolchains.bzl-+++ b/toolchains/built_toolchains.bzl-@@ -28,6 +28,7 @@ _CMAKE_SRCS = {- "3.22.4": [["https://github.com/Kitware/CMake/releases/download/v3.22.4/cmake-3.22.4.tar.gz"], "cmake-3.22.4", "5c55d0b0bc4c191549e3502b8f99a4fe892077611df22b4178cc020626e22a47"],- "3.23.1": [["https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1.tar.gz"], "cmake-3.23.1", "33fd10a8ec687a4d0d5b42473f10459bb92b3ae7def2b745dc10b192760869f3"],- "3.23.2": [["https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2.tar.gz"], "cmake-3.23.2", "f316b40053466f9a416adf981efda41b160ca859e97f6a484b447ea299ff26aa"],-+ "3.25.0": [["https://github.com/Kitware/CMake/releases/download/v3.25.0/cmake-3.25.0.tar.gz"], "cmake-3.25.0", "306463f541555da0942e6f5a0736560f70c487178b9d94a5ae7f34d0538cdd48"],- }-- # buildifier: disable=unnamed-macro-@@ -438,6 +439,18 @@ def _ninja_toolchain(version, register_toolchains):- native.register_toolchains(- "@rules_foreign_cc//toolchains:built_ninja_toolchain",- )-+ if version == "1.11.1":-+ maybe(-+ http_archive,-+ name = "ninja_build_src",-+ build_file_content = _ALL_CONTENT,-+ sha256 = "31747ae633213f1eda3842686f83c2aa1412e0f5691d1c14dbbcc67fe7400cea",-+ strip_prefix = "ninja-1.11.1",-+ urls = [-+ "https://github.com/ninja-build/ninja/archive/v1.11.1.tar.gz",-+ ],-+ )-+ return- if version == "1.11.0":- maybe(- http_archive,-diff --git a/toolchains/prebuilt_toolchains.bzl b/toolchains/prebuilt_toolchains.bzl-index dabfb95..d9c38b4 100644---- a/toolchains/prebuilt_toolchains.bzl-+++ b/toolchains/prebuilt_toolchains.bzl-@@ -67,6 +67,115 @@ def prebuilt_toolchains(cmake_version, ninja_version, register_toolchains):- _make_toolchains(register_toolchains)-- def _cmake_toolchains(version, register_toolchains):-+ if "3.25.0" == version:-+ maybe(-+ http_archive,-+ name = "cmake-3.25.0-linux-aarch64",-+ urls = [-+ "https://github.com/Kitware/CMake/releases/download/v3.25.0/cmake-3.25.0-linux-aarch64.tar.gz",-+ ],-+ sha256 = "27da36d6debe9b30f5c498554ae40cd621a55736f5f2ae2618ed95722a59965a",-+ strip_prefix = "cmake-3.25.0-linux-aarch64",-+ build_file_content = _CMAKE_BUILD_FILE.format(-+ bin = "cmake",-+ env = "{}",-+ ),-+ )-+-+ maybe(-+ http_archive,-+ name = "cmake-3.25.0-linux-x86_64",-+ urls = [-+ "https://github.com/Kitware/CMake/releases/download/v3.25.0/cmake-3.25.0-linux-x86_64.tar.gz",-+ ],-+ sha256 = "ac634d6f0a81d7089adc7be5acff66a6bee3b08615f9a947858ce92a9ef59c8b",-+ strip_prefix = "cmake-3.25.0-linux-x86_64",-+ build_file_content = _CMAKE_BUILD_FILE.format(-+ bin = "cmake",-+ env = "{}",-+ ),-+ )-+-+ maybe(-+ http_archive,-+ name = "cmake-3.25.0-macos-universal",-+ urls = [-+ "https://github.com/Kitware/CMake/releases/download/v3.25.0/cmake-3.25.0-macos-universal.tar.gz",-+ ],-+ sha256 = "c088e761534a2078cd9d0581d39f02d3f9ed05302e33135b55c6d619b263b4c3",-+ strip_prefix = "cmake-3.25.0-macos-universal/CMake.app/Contents",-+ build_file_content = _CMAKE_BUILD_FILE.format(-+ bin = "cmake",-+ env = "{}",-+ ),-+ )-+-+ maybe(-+ http_archive,-+ name = "cmake-3.25.0-windows-i386",-+ urls = [-+ "https://github.com/Kitware/CMake/releases/download/v3.25.0/cmake-3.25.0-windows-i386.zip",-+ ],-+ sha256 = "ddd115257a19ff3dd18fc63f32a00ae742f8b62d2e39bc354629903512f99783",-+ strip_prefix = "cmake-3.25.0-windows-i386",-+ build_file_content = _CMAKE_BUILD_FILE.format(-+ bin = "cmake.exe",-+ env = "{}",-+ ),-+ )-+-+ maybe(-+ http_archive,-+ name = "cmake-3.25.0-windows-x86_64",-+ urls = [-+ "https://github.com/Kitware/CMake/releases/download/v3.25.0/cmake-3.25.0-windows-x86_64.zip",-+ ],-+ sha256 = "b46030c10cab1170355952f9ac59f7e6dabc248070fc53f15dff11d4ed2910f8",-+ strip_prefix = "cmake-3.25.0-windows-x86_64",-+ build_file_content = _CMAKE_BUILD_FILE.format(-+ bin = "cmake.exe",-+ env = "{}",-+ ),-+ )-+-+ # buildifier: leave-alone-+ maybe(-+ prebuilt_toolchains_repository,-+ name = "cmake_3.25.0_toolchains",-+ repos = {-+ "cmake-3.25.0-linux-aarch64": [-+ "@platforms//cpu:aarch64",-+ "@platforms//os:linux",-+ ],-+ "cmake-3.25.0-linux-x86_64": [-+ "@platforms//cpu:x86_64",-+ "@platforms//os:linux",-+ ],-+ "cmake-3.25.0-macos-universal": [-+ "@platforms//os:macos",-+ ],-+ "cmake-3.25.0-windows-i386": [-+ "@platforms//cpu:x86_32",-+ "@platforms//os:windows",-+ ],-+ "cmake-3.25.0-windows-x86_64": [-+ "@platforms//cpu:x86_64",-+ "@platforms//os:windows",-+ ],-+ },-+ tool = "cmake",-+ )-+-+ if register_toolchains:-+ native.register_toolchains(-+ "@cmake_3.25.0_toolchains//:cmake-3.25.0-linux-aarch64_toolchain",-+ "@cmake_3.25.0_toolchains//:cmake-3.25.0-linux-x86_64_toolchain",-+ "@cmake_3.25.0_toolchains//:cmake-3.25.0-macos-universal_toolchain",-+ "@cmake_3.25.0_toolchains//:cmake-3.25.0-windows-i386_toolchain",-+ "@cmake_3.25.0_toolchains//:cmake-3.25.0-windows-x86_64_toolchain",-+ )-+-+ return- if "3.23.2" == version:- maybe(- http_archive,-@@ -4196,6 +4305,78 @@ def _cmake_toolchains(version, register_toolchains):- fail("Unsupported version: " + str(version))-- def _ninja_toolchains(version, register_toolchains):-+ if "1.11.1" == version:-+ maybe(-+ http_archive,-+ name = "ninja_1.11.1_linux",-+ urls = [-+ "https://github.com/ninja-build/ninja/releases/download/v1.11.1/ninja-linux.zip",-+ ],-+ sha256 = "b901ba96e486dce377f9a070ed4ef3f79deb45f4ffe2938f8e7ddc69cfb3df77",-+ strip_prefix = "",-+ build_file_content = _NINJA_BUILD_FILE.format(-+ bin = "ninja",-+ env = "{\"NINJA\": \"$(execpath :ninja_bin)\"}",-+ ),-+ )-+-+ maybe(-+ http_archive,-+ name = "ninja_1.11.1_mac",-+ urls = [-+ "https://github.com/ninja-build/ninja/releases/download/v1.11.1/ninja-mac.zip",-+ ],-+ sha256 = "482ecb23c59ae3d4f158029112de172dd96bb0e97549c4b1ca32d8fad11f873e",-+ strip_prefix = "",-+ build_file_content = _NINJA_BUILD_FILE.format(-+ bin = "ninja",-+ env = "{\"NINJA\": \"$(execpath :ninja_bin)\"}",-+ ),-+ )-+-+ maybe(-+ http_archive,-+ name = "ninja_1.11.1_win",-+ urls = [-+ "https://github.com/ninja-build/ninja/releases/download/v1.11.1/ninja-win.zip",-+ ],-+ sha256 = "524b344a1a9a55005eaf868d991e090ab8ce07fa109f1820d40e74642e289abc",-+ strip_prefix = "",-+ build_file_content = _NINJA_BUILD_FILE.format(-+ bin = "ninja.exe",-+ env = "{\"NINJA\": \"$(execpath :ninja_bin)\"}",-+ ),-+ )-+-+ # buildifier: leave-alone-+ maybe(-+ prebuilt_toolchains_repository,-+ name = "ninja_1.11.1_toolchains",-+ repos = {-+ "ninja_1.11.1_linux": [-+ "@platforms//cpu:x86_64",-+ "@platforms//os:linux",-+ ],-+ "ninja_1.11.1_mac": [-+ "@platforms//cpu:x86_64",-+ "@platforms//os:macos",-+ ],-+ "ninja_1.11.1_win": [-+ "@platforms//cpu:x86_64",-+ "@platforms//os:windows",-+ ],-+ },-+ tool = "ninja",-+ )-+-+ if register_toolchains:-+ native.register_toolchains(-+ "@ninja_1.11.1_toolchains//:ninja_1.11.1_linux_toolchain",-+ "@ninja_1.11.1_toolchains//:ninja_1.11.1_mac_toolchain",-+ "@ninja_1.11.1_toolchains//:ninja_1.11.1_win_toolchain",-+ )-+-+ return- if "1.11.0" == version:- maybe(- http_archive,-diff --git a/toolchains/prebuilt_toolchains.py b/toolchains/prebuilt_toolchains.py-index 5288b27..a193021 100755---- a/toolchains/prebuilt_toolchains.py-+++ b/toolchains/prebuilt_toolchains.py-@@ -10,6 +10,7 @@ CMAKE_SHA256_URL_TEMPLATE = "https://cmake.org/files/v{minor}/cmake-{full}-SHA-2- CMAKE_URL_TEMPLATE = "https://github.com/Kitware/CMake/releases/download/v{full}/{file}"-- CMAKE_VERSIONS = [-+ "3.25.0",- "3.23.2",- "3.23.1",- "3.22.4",-@@ -116,6 +117,7 @@ NINJA_TARGETS = {- }-- NINJA_VERSIONS = (-+ "1.11.1",- "1.10.2",- "1.10.1",- "1.10.0",diff --git a/cmake/patch.sh b/cmake/patch.shnew file mode 100644index 00000000..410c1254--- /dev/null+++ b/cmake/patch.sh@@ -0,0 +1,18 @@+#! /bin/sh++set +x+set -euo pipefail+++patch="$1"; shift++# ignore the error if the patch is already applied+if ! out=$(patch -p1 -N -r "rejects.bin" < "$patch")+then+ echo "$out" | grep -q "Reversed (or previously applied) patch detected! Skipping patch."+ test -s "rejects.bin" # Make sure we have rejects.+else+ test -f "rejects.bin" && ! test -s "rejects.bin" # Make sure we have no rejects.+fi++rm -f "rejects.bin"\ No newline at end of filediff --git a/include/collection.h b/include/collection.hindex ab847a4c..8d77cfde 100644--- a/include/collection.h+++ b/include/collection.h@@ -463,14 +463,6 @@ public:Option<bool> validate_reference_filter(const std::string& filter_query) const;- Option<std::string> get_reference_field(const std::string & collection_name) const;-- Option<bool> get_reference_filter_ids(const std::string & filter_query,- filter_result_t& filter_result,- const std::string & collection_name) const;-- Option<bool> validate_reference_filter(const std::string& filter_query) const;-Option<nlohmann::json> get(const std::string & id) const;Option<std::string> remove(const std::string & id, bool remove_from_store = true);diff --git a/include/field.h b/include/field.hindex 63feff0f..305675ab 100644--- a/include/field.h+++ b/include/field.h@@ -411,7 +411,7 @@ struct field {static Option<bool> json_field_to_field(bool enable_nested_fields, nlohmann::json& field_json,std::vector<field>& the_fields,- string& fallback_field_type, size_t& num_auto_detect_fields);+ string& fallback_field_type, size_t& num_auto_detect_fields,const nlohmann::json& all_fields_json = nlohmann::json());static Option<bool> json_fields_to_fields(bool enable_nested_fields,nlohmann::json& fields_json,@@ -475,7 +475,7 @@ struct field {}auto op = json_field_to_field(enable_nested_fields,- field_json, the_fields, fallback_field_type, num_auto_detect_fields);+ field_json, the_fields, fallback_field_type, num_auto_detect_fields, fields_json);if(!op.ok()) {return op;}diff --git a/src/collection.cpp b/src/collection.cppindex 45ae966a..af35b70d 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -246,6 +246,7 @@ nlohmann::json Collection::get_summary_json() const {field_json[fields::reference] = coll_field.reference;}+fields_arr.push_back(field_json);}@@ -2580,71 +2581,6 @@ Option<bool> Collection::validate_reference_filter(const std::string& filter_quereturn Option<bool>(true);}-Option<std::string> Collection::get_reference_field(const std::string & collection_name) const {- std::shared_lock lock(mutex);-- std::string reference_field_name;- for (auto const& pair: reference_fields) {- auto reference_pair = pair.second;- if (reference_pair.collection == collection_name) {- reference_field_name = reference_pair.field;- break;- }- }-- if (reference_field_name.empty()) {- return Option<std::string>(400, "Could not find any field in `" + name + "` referencing the collection `"- + collection_name + "`.");- }-- return Option(reference_field_name);-}--Option<bool> Collection::get_reference_filter_ids(const std::string & filter_query,- filter_result_t& filter_result,- const std::string & collection_name) const {- auto reference_field_op = get_reference_field(collection_name);- if (!reference_field_op.ok()) {- return Option<bool>(reference_field_op.code(), reference_field_op.error());- }-- std::shared_lock lock(mutex);-- const std::string doc_id_prefix = std::to_string(collection_id) + "_" + DOC_ID_PREFIX + "_";- filter_node_t* filter_tree_root = nullptr;- Option<bool> parse_op = filter::parse_filter_query(filter_query, search_schema,- store, doc_id_prefix, filter_tree_root);- if(!parse_op.ok()) {- return parse_op;- }-- // Reference helper field has the sequence id of other collection's documents.- auto field_name = reference_field_op.get() + REFERENCE_HELPER_FIELD_SUFFIX;- auto filter_op = index->do_reference_filtering_with_lock(filter_tree_root, filter_result, field_name);- if (!filter_op.ok()) {- return filter_op;- }-- delete filter_tree_root;- return Option<bool>(true);-}--Option<bool> Collection::validate_reference_filter(const std::string& filter_query) const {- std::shared_lock lock(mutex);-- const std::string doc_id_prefix = std::to_string(collection_id) + "_" + DOC_ID_PREFIX + "_";- filter_node_t* filter_tree_root = nullptr;- Option<bool> filter_op = filter::parse_filter_query(filter_query, search_schema,- store, doc_id_prefix, filter_tree_root);-- if(!filter_op.ok()) {- return filter_op;- }-- delete filter_tree_root;- return Option<bool>(true);-}-bool Collection::facet_value_to_string(const facet &a_facet, const facet_count_t &facet_count,const nlohmann::json &document, std::string &value) const {diff --git a/src/field.cpp b/src/field.cppindex 129c7512..be2000b0 100644--- a/src/field.cpp+++ b/src/field.cpp@@ -523,7 +523,7 @@ Option<bool> filter::parse_filter_query(const std::string& filter_query,Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::json& field_json,std::vector<field>& the_fields,- string& fallback_field_type, size_t& num_auto_detect_fields) {+ string& fallback_field_type, size_t& num_auto_detect_fields, const nlohmann::json& all_fields_json) {if(field_json["name"] == "id") {// No field should exist with the name "id" as it is reserved for internal use