Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Intersection of documents across the search tokens.

Kishore Nallan566c4ce666c1
diff --git a/CMakeLists.txt b/CMakeLists.txt
index faffab03..19ac62f6 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -7,5 +7,5 @@ include_directories(include)
include_directories(external/for)
-add_executable(search external/for/libfor.a src/art.cpp src/main.cpp)
+add_executable(search external/for/libfor.a src/art.cpp src/intersection.cpp src/main.cpp)
target_link_libraries(search for)
diff --git a/include/forarray.h b/include/forarray.h
index 3d1cdb7b..0da7df78 100644
--- a/include/forarray.h
+++ b/include/forarray.h
@@ -61,6 +61,12 @@ public:
return actual == value;
}
+ uint32_t* uncompress() {
+ uint32_t *out = new uint32_t[length];
+ for_uncompress(in, out, length);
+ return out;
+ }
+
uint32_t getSizeInBytes() {
return size_bytes;
}
diff --git a/include/intersection.h b/include/intersection.h
new file mode 100644
index 00000000..cf7f38c2
--- /dev/null
+++ b/include/intersection.h
@@ -0,0 +1,13 @@
+#pragma once
+
+#include <cstddef>
+#include <stdint.h>
+
+/* Different intersection routines adapted from:
+ * https://github.com/lemire/SIMDCompressionAndIntersection/blob/master/src/intersection.cpp
+ */
+class Intersection {
+public:
+ // Fast scalar scheme designed by N. Kurz. Returns the size of out (intersected set)
+ static size_t scalar(const uint32_t *A, const size_t lenA, const uint32_t *B, const size_t lenB, uint32_t *out);
+};
\ No newline at end of file
diff --git a/src/intersection.cpp b/src/intersection.cpp
new file mode 100644
index 00000000..3dd9e95a
--- /dev/null
+++ b/src/intersection.cpp
@@ -0,0 +1,32 @@
+#include "intersection.h"
+
+size_t Intersection::scalar(const uint32_t *A, const size_t lenA,
+ const uint32_t *B, const size_t lenB, uint32_t *out) {
+ const uint32_t *const initout(out);
+ if (lenA == 0 || lenB == 0)
+ return 0;
+
+ const uint32_t *endA = A + lenA;
+ const uint32_t *endB = B + lenB;
+
+ while (1) {
+ while (*A < *B) {
+ SKIP_FIRST_COMPARE:
+ if (++A == endA)
+ return (out - initout);
+ }
+ while (*A > *B) {
+ if (++B == endB)
+ return (out - initout);
+ }
+ if (*A == *B) {
+ *out++ = *A;
+ if (++A == endA || ++B == endB)
+ return (out - initout);
+ } else {
+ goto SKIP_FIRST_COMPARE;
+ }
+ }
+
+ return (out - initout); // NOTREACHED
+}
diff --git a/src/main.cpp b/src/main.cpp
index fd893a9e..cab1f31a 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -12,6 +12,7 @@
#include "art.h"
#include "topster.h"
#include "forarray.h"
+#include "intersection.h"
#include "util.h"
using namespace std;
@@ -127,11 +128,30 @@ void find_documents(art_tree & t, string q) {
q = div(q.quot, token_leaves[i].size());
u[i] = token_leaves[i][q.rem];
}
- // Do what you want here with u.
+
for(art_leaf* x : u) {
- cout << x->key << ' ';
+ cout << x->key << ' ';
+ }
+
+ // sort based on matched document size to perform effective intersection
+ sort(u.begin(), u.end(), [](const art_leaf* left, const art_leaf* right) {
+ return left->values->ids.getLength() < right->values->ids.getLength();
+ });
+
+ uint32_t* result = u[0]->values->ids.uncompress();
+ size_t result_size = u[0]->values->ids.getLength();
+
+ for(auto i=1; i<u.size(); i++) {
+ uint32_t* out = new uint32_t[result_size];
+ uint32_t* curr = u[i]->values->ids.uncompress();
+ result_size = Intersection::scalar(result, result_size, curr, u[i]->values->ids.getLength(), out);
+ delete result;
+ delete curr;
+ result = out;
}
- cout << '\n';
+
+ cout << "RESULT SIZE: " << result_size << endl;
+ delete result;
}
}