Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Intersection of documents across the search tokens.
Kishore Nallan566c4ce666c1
diff --git a/CMakeLists.txt b/CMakeLists.txtindex faffab03..19ac62f6 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -7,5 +7,5 @@ include_directories(include)include_directories(external/for)-add_executable(search external/for/libfor.a src/art.cpp src/main.cpp)+add_executable(search external/for/libfor.a src/art.cpp src/intersection.cpp src/main.cpp)target_link_libraries(search for)diff --git a/include/forarray.h b/include/forarray.hindex 3d1cdb7b..0da7df78 100644--- a/include/forarray.h+++ b/include/forarray.h@@ -61,6 +61,12 @@ public:return actual == value;}+ uint32_t* uncompress() {+ uint32_t *out = new uint32_t[length];+ for_uncompress(in, out, length);+ return out;+ }+uint32_t getSizeInBytes() {return size_bytes;}diff --git a/include/intersection.h b/include/intersection.hnew file mode 100644index 00000000..cf7f38c2--- /dev/null+++ b/include/intersection.h@@ -0,0 +1,13 @@+#pragma once++#include <cstddef>+#include <stdint.h>++/* Different intersection routines adapted from:+ * https://github.com/lemire/SIMDCompressionAndIntersection/blob/master/src/intersection.cpp+ */+class Intersection {+public:+ // Fast scalar scheme designed by N. Kurz. Returns the size of out (intersected set)+ static size_t scalar(const uint32_t *A, const size_t lenA, const uint32_t *B, const size_t lenB, uint32_t *out);+};\ No newline at end of filediff --git a/src/intersection.cpp b/src/intersection.cppnew file mode 100644index 00000000..3dd9e95a--- /dev/null+++ b/src/intersection.cpp@@ -0,0 +1,32 @@+#include "intersection.h"++size_t Intersection::scalar(const uint32_t *A, const size_t lenA,+ const uint32_t *B, const size_t lenB, uint32_t *out) {+ const uint32_t *const initout(out);+ if (lenA == 0 || lenB == 0)+ return 0;++ const uint32_t *endA = A + lenA;+ const uint32_t *endB = B + lenB;++ while (1) {+ while (*A < *B) {+ SKIP_FIRST_COMPARE:+ if (++A == endA)+ return (out - initout);+ }+ while (*A > *B) {+ if (++B == endB)+ return (out - initout);+ }+ if (*A == *B) {+ *out++ = *A;+ if (++A == endA || ++B == endB)+ return (out - initout);+ } else {+ goto SKIP_FIRST_COMPARE;+ }+ }++ return (out - initout); // NOTREACHED+}diff --git a/src/main.cpp b/src/main.cppindex fd893a9e..cab1f31a 100644--- a/src/main.cpp+++ b/src/main.cpp@@ -12,6 +12,7 @@#include "art.h"#include "topster.h"#include "forarray.h"+#include "intersection.h"#include "util.h"using namespace std;@@ -127,11 +128,30 @@ void find_documents(art_tree & t, string q) {q = div(q.quot, token_leaves[i].size());u[i] = token_leaves[i][q.rem];}- // Do what you want here with u.+for(art_leaf* x : u) {- cout << x->key << ' ';+ cout << x->key << ' ';+ }++ // sort based on matched document size to perform effective intersection+ sort(u.begin(), u.end(), [](const art_leaf* left, const art_leaf* right) {+ return left->values->ids.getLength() < right->values->ids.getLength();+ });++ uint32_t* result = u[0]->values->ids.uncompress();+ size_t result_size = u[0]->values->ids.getLength();++ for(auto i=1; i<u.size(); i++) {+ uint32_t* out = new uint32_t[result_size];+ uint32_t* curr = u[i]->values->ids.uncompress();+ result_size = Intersection::scalar(result, result_size, curr, u[i]->values->ids.getLength(), out);+ delete result;+ delete curr;+ result = out;}- cout << '\n';++ cout << "RESULT SIZE: " << result_size << endl;+ delete result;}}