Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

RocksDB based persistence.

Kishore Nallan44da808f16a6
diff --git a/CMakeLists.txt b/CMakeLists.txt
index b0137695..cca77389 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -14,5 +14,5 @@ include_directories(/usr/local/opt/openssl/include)
add_executable(search src/art.cpp src/intersection.cpp src/main.cpp src/collection.cpp)
add_executable(typesense-server src/art.cpp src/intersection.cpp src/collection.cpp src/server.cpp)
-target_link_libraries(search ${CMAKE_SOURCE_DIR}/external/for/libfor.a)
-target_link_libraries(typesense-server ${CMAKE_SOURCE_DIR}/external/for/libfor.a curl h2o-evloop pthread /usr/local/opt/openssl/lib/libssl.a /usr/local/opt/openssl/lib/libcrypto.a)
\ No newline at end of file
+target_link_libraries(search ${CMAKE_SOURCE_DIR}/external/for/libfor.a rocksdb)
+target_link_libraries(typesense-server ${CMAKE_SOURCE_DIR}/external/for/libfor.a curl h2o-evloop pthread /usr/local/opt/openssl/lib/libssl.a /usr/local/opt/openssl/lib/libcrypto.a rocksdb)
\ No newline at end of file
diff --git a/include/collection_state.h b/include/collection_state.h
deleted file mode 100644
index 68f381f3..00000000
--- a/include/collection_state.h
+++ /dev/null
@@ -1,20 +0,0 @@
-#pragma once
-
-#include <stdint.h>
-
-/*
- * Stores all information about a collection.
- * Uses RocksDB for persistence.
- */
-class CollectionState {
-
-private:
-
- uint32_t id = 0;
-
-public:
-
- uint32_t nextId() {
- return ++id;
- }
-};
\ No newline at end of file
diff --git a/include/sole.hpp b/include/sole.hpp
new file mode 100644
index 00000000..94ad74c6
--- /dev/null
+++ b/include/sole.hpp
@@ -0,0 +1,778 @@
+/* Sole is a lightweight C++11 library to generate universally unique identificators.
+ * Sole provides interface for UUID versions 0, 1 and 4.
+
+ * https://github.com/r-lyeh/sole
+ * Copyright (c) 2013,2014,2015 r-lyeh. zlib/libpng licensed.
+
+ * Based on code by Dmitri Bouianov, Philip O'Toole, Poco C++ libraries and anonymous
+ * code found on the net. Thanks guys!
+
+ * Theory: (see Hoylen's answer at [1])
+ * - UUID version 1 (48-bit MAC address + 60-bit clock with a resolution of 100ns)
+ * Clock wraps in 3603 A.D.
+ * Up to 10000000 UUIDs per second.
+ * MAC address revealed.
+ *
+ * - UUID Version 4 (122-bits of randomness)
+ * See [2] or other analysis that describe how very unlikely a duplicate is.
+ *
+ * - Use v1 if you need to sort or classify UUIDs per machine.
+ * Use v1 if you are worried about leaving it up to probabilities (e.g. your are the
+ * type of person worried about the earth getting destroyed by a large asteroid in your
+ * lifetime). Just use a v1 and it is guaranteed to be unique till 3603 AD.
+ *
+ * - Use v4 if you are worried about security issues and determinism. That is because
+ * v1 UUIDs reveal the MAC address of the machine it was generated on and they can be
+ * predictable. Use v4 if you need more than 10 million uuids per second, or if your
+ * application wants to live past 3603 A.D.
+
+ * Additionally a custom UUID v0 is provided:
+ * - 16-bit PID + 48-bit MAC address + 60-bit clock with a resolution of 100ns since Unix epoch
+ * - Format is EPOCH_LOW-EPOCH_MID-VERSION(0)|EPOCH_HI-PID-MAC
+ * - Clock wraps in 3991 A.D.
+ * - Up to 10000000 UUIDs per second.
+ * - MAC address and PID revealed.
+
+ * References:
+ * - [1] http://stackoverflow.com/questions/1155008/how-unique-is-uuid
+ * - [2] http://en.wikipedia.org/wiki/UUID#Random%5FUUID%5Fprobability%5Fof%5Fduplicates
+ * - http://en.wikipedia.org/wiki/Universally_unique_identifier
+ * - http://en.cppreference.com/w/cpp/numeric/random/random_device
+ * - http://www.itu.int/ITU-T/asn1/uuid.html f81d4fae-7dec-11d0-a765-00a0c91e6bf6
+
+ * - rlyeh ~~ listening to Hedon Cries / Until The Sun Goes up
+ */
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+#pragma once
+#include <stdint.h>
+#include <stdio.h> // for size_t; should be stddef.h instead; however, clang+archlinux fails when compiling it (@Travis-Ci)
+#include <sys/types.h> // for uint32_t; should be stdint.h instead; however, GCC 5 on OSX fails when compiling it (See issue #11)
+#include <functional>
+#include <string>
+
+// public API
+
+#define SOLE_VERSION "1.0.0" // (2016/02/03): Initial semver adherence; Switch to header-only; Remove warnings
+
+namespace sole
+{
+ // 128-bit basic UUID type that allows comparison and sorting.
+ // Use .str() for printing and .pretty() for pretty printing.
+ // Also, ostream friendly.
+ struct uuid
+ {
+ uint64_t ab;
+ uint64_t cd;
+
+ bool operator==( const uuid &other ) const;
+ bool operator!=( const uuid &other ) const;
+ bool operator <( const uuid &other ) const;
+
+ std::string pretty() const;
+ std::string base62() const;
+ std::string str() const;
+
+ template<typename ostream>
+ inline friend ostream &operator<<( ostream &os, const uuid &self ) {
+ return os << self.str(), os;
+ }
+ };
+
+ // Generators
+ uuid uuid0(); // UUID v0, pro: unique; cons: MAC revealed, pid revealed, predictable.
+ uuid uuid1(); // UUID v1, pro: unique; cons: MAC revealed, predictable.
+ uuid uuid4(); // UUID v4, pros: anonymous, fast; con: uuids "can clash"
+
+ // Rebuilders
+ uuid rebuild( uint64_t ab, uint64_t cd );
+ uuid rebuild( const std::string &uustr );
+}
+
+#ifdef _MSC_VER
+#pragma warning(push)
+#pragma warning(disable:4127)
+#endif
+
+namespace std {
+ template<>
+ class hash< sole::uuid > : public std::unary_function< sole::uuid, size_t > {
+ public:
+ // hash functor: hash uuid to size_t value by pseudorandomizing transform
+ size_t operator()( const sole::uuid &uuid ) const {
+ if( sizeof(size_t) > 4 ) {
+ return size_t( uuid.ab ^ uuid.cd );
+ } else {
+ uint64_t hash64 = uuid.ab ^ uuid.cd;
+ return size_t( uint32_t( hash64 >> 32 ) ^ uint32_t( hash64 ) );
+ }
+ }
+ };
+}
+
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+
+// implementation
+
+#include <memory.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <time.h>
+
+#include <cstring>
+#include <ctime>
+
+#include <iomanip>
+#include <random>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#if defined(_WIN32)
+# include <winsock2.h>
+# include <process.h>
+# include <iphlpapi.h>
+# pragma comment(lib,"iphlpapi.lib")
+# define $windows $yes
+#elif defined(__FreeBSD__) || defined(__NetBSD__) || \
+ defined(__OpenBSD__) || defined(__MINT__) || defined(__bsdi__)
+# include <ifaddrs.h>
+# include <net/if_dl.h>
+# include <sys/socket.h>
+# include <sys/time.h>
+# include <sys/types.h>
+# include <unistd.h>
+# define $bsd $yes
+#elif (defined(__APPLE__) && defined(__MACH__))
+# include <ifaddrs.h>
+# include <net/if_dl.h>
+# include <sys/socket.h>
+# include <sys/time.h>
+# include <sys/types.h>
+# include <unistd.h>
+# pragma clang diagnostic push
+# pragma clang diagnostic ignored "-Wdollar-in-identifier-extension"
+# define $osx $yes
+#elif defined(__linux__)
+# include <arpa/inet.h>
+# include <net/if.h>
+# include <netinet/in.h>
+# include <sys/ioctl.h>
+# include <sys/socket.h>
+# include <sys/time.h>
+# include <unistd.h>
+# define $linux $yes
+#else //elif defined(__unix__)
+# if defined(__VMS)
+# include <ioctl.h>
+# include <inet.h>
+# else
+# include <sys/ioctl.h>
+# include <arpa/inet.h>
+# endif
+# if defined(sun) || defined(__sun)
+# include <sys/sockio.h>
+# endif
+# include <net/if.h>
+# include <net/if_arp.h>
+# include <netdb.h>
+# include <netinet/in.h>
+# include <sys/socket.h>
+# include <sys/time.h>
+# include <sys/types.h>
+# include <unistd.h>
+# if defined(__VMS)
+ namespace { enum { MAXHOSTNAMELEN = 64 }; }
+# endif
+# define $unix $yes
+#endif
+
+#ifdef _MSC_VER
+# define $msvc $yes
+#endif
+
+#if defined(__GNUC__) && (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 < 50100)
+namespace std
+{
+ static inline std::string put_time( const std::tm* tmb, const char* fmt ) {
+ std::string s( 128, '\0' );
+ while( !strftime( &s[0], s.size(), fmt, tmb ) )
+ s.resize( s.size() + 128 );
+ return s;
+ }
+}
+#endif
+
+////////////////////////////////////////////////////////////////////////////////////
+
+#ifdef $windows
+#define $welse $no
+#else
+#define $windows $no
+#define $welse $yes
+#endif
+
+#ifdef $bsd
+#define $belse $no
+#else
+#define $bsd $no
+#define $belse $yes
+#endif
+
+#ifdef $linux
+#define $lelse $no
+#else
+#define $linux $no
+#define $lelse $yes
+#endif
+
+#ifdef $unix
+#define $uelse $no
+#else
+#define $unix $no
+#define $uelse $yes
+#endif
+
+#ifdef $osx
+#define $oelse $no
+#else
+#define $osx $no
+#define $oelse $yes
+#endif
+
+#ifdef $msvc
+#define $melse $no
+#else
+#define $msvc $no
+#define $melse $yes
+#endif
+
+#define $yes(...) __VA_ARGS__
+#define $no(...)
+
+inline bool sole::uuid::operator==( const sole::uuid &other ) const {
+ return ab == other.ab && cd == other.cd;
+}
+inline bool sole::uuid::operator!=( const sole::uuid &other ) const {
+ return !operator==(other);
+}
+inline bool sole::uuid::operator<( const sole::uuid &other ) const {
+ if( ab < other.ab ) return true;
+ if( ab > other.ab ) return false;
+ if( cd < other.cd ) return true;
+ return false;
+}
+
+namespace sole {
+
+ inline std::string rebase( uint64_t input, const std::string &basemap ) {
+ uint64_t rem, size = basemap.size();
+ std::string res;
+ do {
+ rem = input % size;
+ res = std::string() + basemap[int(rem)] + res;
+ input /= size;
+ } while (input > 0);
+ return res;
+ }
+
+ inline uint64_t rebase( const std::string &input, const std::string &basemap ) {
+ auto strpos = [](const std::string &chars, char ch ) -> size_t {
+ return chars.find_first_of( ch );
+ };
+ auto limit = input.size();
+ auto size = basemap.size();
+ uint64_t res = strpos( basemap, input[0] );
+ for( size_t i = 1; i < limit; ++i )
+ res = size * res + strpos( basemap, input[i] );
+ return res;
+ }
+
+ inline std::string printftime( uint64_t timestamp_secs = 0, const std::string &locale = std::string() ) {
+ std::string timef;
+ try {
+ // Taken from parameter
+ //std::string locale; // = "es-ES", "Chinese_China.936", "en_US.UTF8", etc...
+ std::time_t t = timestamp_secs;
+ std::tm tm;
+ $msvc(
+ localtime_s( &tm, &t );
+ )
+ $melse(
+ localtime_r( &t, &tm );
+ )
+
+ std::stringstream ss;
+
+ std::locale lc( locale.c_str() );
+ ss.imbue( lc );
+ ss << std::put_time( &tm, "\"%c\"" );
+
+ timef = ss.str();
+ }
+ catch(...) {
+ timef = "\"\"";
+ }
+ return timef;
+ }
+
+ inline std::string uuid::pretty() const {
+ std::stringstream ss;
+
+ uint64_t a = (ab >> 32);
+ uint64_t b = (ab & 0xFFFFFFFF);
+ uint64_t c = (cd >> 32);
+ uint64_t d = (cd & 0xFFFFFFFF);
+
+ int version = (b & 0xF000) >> 12;
+ uint64_t timestamp = ((b & 0x0FFF) << 48 ) | (( b >> 16 ) << 32) | a; // in 100ns units
+
+ ss << "version=" << (version) << ',';
+
+ if( version == 1 )
+ timestamp = timestamp - 0x01b21dd213814000ULL; // decrement Gregorian calendar
+
+ ss << std::hex << std::nouppercase << std::setfill('0');
+ version <= 1 && ss << "timestamp=" << printftime(timestamp/10000000) << ',';
+ version <= 1 && ss << "mac=" << std::setw(4) << (c & 0xFFFF) << std::setw(8) << d << ',';
+ version == 4 && ss << "randbits=" << std::setw(8) << (ab & 0xFFFFFFFFFFFF0FFFULL) << std::setw(8) << (cd & 0x3FFFFFFFFFFFFFFFULL) << ',';
+
+ ss << std::dec;
+ version == 0 && ss << "pid=" << std::setw(4) << (c >> 16 ) << ',';
+ version == 1 && ss << "clock_seq=" << std::setw(4) << ((c >> 16) & 0x3FFF) << ',';
+
+ return ss.str();
+ }
+
+ inline std::string uuid::str() const {
+ std::stringstream ss;
+ ss << std::hex << std::nouppercase << std::setfill('0');
+
+ uint32_t a = (ab >> 32);
+ uint32_t b = (ab & 0xFFFFFFFF);
+ uint32_t c = (cd >> 32);
+ uint32_t d = (cd & 0xFFFFFFFF);
+
+ ss << std::setw(8) << (a) << '-';
+ ss << std::setw(4) << (b >> 16) << '-';
+ ss << std::setw(4) << (b & 0xFFFF) << '-';
+ ss << std::setw(4) << (c >> 16 ) << '-';
+ ss << std::setw(4) << (c & 0xFFFF);
+ ss << std::setw(8) << d;
+
+ return ss.str();
+ }
+
+ inline std::string uuid::base62() const {
+ const char *base62 =
+ "0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+ "abcdefghijklmnopqrstuvwxyz";
+ return rebase( ab, base62 ) + "-" + rebase( cd, base62 );
+ }
+
+ //////////////////////////////////////////////////////////////////////////////////////
+ // multiplatform clock_gettime()
+
+ $windows(
+ struct timespec {
+ uint64_t tv_sec;
+ uint64_t tv_nsec;
+ };
+ struct timezone {
+ int tz_minuteswest; /* minutes W of Greenwich */
+ int tz_dsttime; /* type of dst correction */
+ };
+ inline int gettimeofday( struct timeval *tv, struct timezone *tz ) {
+ FILETIME ft;
+ uint64_t tmpres = 0;
+
+ if( NULL != tv ) {
+ GetSystemTimeAsFileTime(&ft);
+
+ // The GetSystemTimeAsFileTime returns the number of 100 nanosecond
+ // intervals since Jan 1, 1601 in a structure. Copy the high bits to
+ // the 64 bit tmpres, shift it left by 32 then or in the low 32 bits.
+ tmpres |= ft.dwHighDateTime;
+ tmpres <<= 32;
+ tmpres |= ft.dwLowDateTime;
+
+ // Convert to microseconds by dividing by 10
+ tmpres /= 10;
+
+ // The Unix epoch starts on Jan 1 1970. Need to subtract the difference
+ // in seconds from Jan 1 1601.
+ tmpres -= 11644473600000000ULL;
+
+ // Finally change microseconds to seconds and place in the seconds value.
+ // The modulus picks up the microseconds.
+ tv->tv_sec = static_cast<long>(tmpres / 1000000UL);
+ tv->tv_usec = (tmpres % 1000000UL);
+ }
+
+ if( NULL != tz ) {
+ static bool once = true;
+ if( once ) {
+ once = false;
+ _tzset();
+ }
+
+ long timezoneSecs = 0;
+ int daylight = 0;
+
+ $msvc(
+ _get_timezone(&timezoneSecs);
+ _get_daylight(&daylight);
+ )
+ $melse(
+ timezoneSecs = _timezone;
+ daylight = _daylight;
+ )
+
+ tz->tz_minuteswest = timezoneSecs / 60;
+ tz->tz_dsttime = daylight;
+ }
+
+ return 0;
+ }
+ )
+ $lelse( $belse( // if not linux, if not bsd... valid for apple/win32
+ inline int clock_gettime( int /*clk_id*/, struct timespec* t ) {
+ struct timeval now;
+ int rv = gettimeofday(&now, NULL);
+ if( rv ) return rv;
+ t->tv_sec = now.tv_sec;
+ t->tv_nsec = now.tv_usec * 1000;
+ return 0;
+ }
+ ))
+
+ //////////////////////////////////////////////////////////////////////////////////////
+ // Timestamp and MAC interfaces
+
+ // Returns number of 100ns intervals
+ inline uint64_t get_time( uint64_t offset ) {
+ struct timespec tp;
+ clock_gettime(0 /*CLOCK_REALTIME*/, &tp);
+
+ // Convert to 100-nanosecond intervals
+ uint64_t uuid_time;
+ uuid_time = tp.tv_sec * 10000000;
+ uuid_time = uuid_time + (tp.tv_nsec / 100);
+ uuid_time = uuid_time + offset;
+
+ // If the clock looks like it went backwards, or is the same, increment it.
+ static uint64_t last_uuid_time = 0;
+ if( last_uuid_time > uuid_time )
+ last_uuid_time = uuid_time;
+ else
+ last_uuid_time = ++uuid_time;
+
+ return uuid_time;
+ }
+
+ // Looks for first MAC address of any network device, any size.
+ inline bool get_any_mac( std::vector<unsigned char> &_node ) {
+ $windows({
+ PIP_ADAPTER_INFO pAdapterInfo;
+ PIP_ADAPTER_INFO pAdapter = 0;
+ ULONG len = sizeof(IP_ADAPTER_INFO);
+ pAdapterInfo = reinterpret_cast<IP_ADAPTER_INFO*>(new char[len]);
+
+ // Make an initial call to GetAdaptersInfo to get
+ // the necessary size into len
+ DWORD rc = GetAdaptersInfo(pAdapterInfo, &len);
+ if (rc == ERROR_BUFFER_OVERFLOW)
+ {
+ delete [] reinterpret_cast<char*>(pAdapterInfo);
+ pAdapterInfo = reinterpret_cast<IP_ADAPTER_INFO*>(new char[len]);
+ }
+ else if (rc != ERROR_SUCCESS)
+ {
+ return $no("cannot get network adapter list") false;
+ }
+
+ bool found = false, gotten = false;
+ if (GetAdaptersInfo(pAdapterInfo, &len) == NO_ERROR)
+ {
+ gotten = true;
+
+ pAdapter = pAdapterInfo;
+ while (pAdapter && !found)
+ {
+ if (pAdapter->Type == MIB_IF_TYPE_ETHERNET && pAdapter->AddressLength > 0 )
+ {
+ _node.resize( pAdapter->AddressLength );
+ std::memcpy(_node.data(), pAdapter->Address, _node.size() );
+ found = true;
+ }
+ pAdapter = pAdapter->Next;
+ }
+ }
+
+ delete [] reinterpret_cast<char*>(pAdapterInfo);
+
+ if( !gotten )
+ return $no("cannot get network adapter list") false;
+
+ if (!found)
+ return $no("no Ethernet adapter found") false;
+
+ return true;
+ })
+
+ $bsd({
+ struct ifaddrs* ifaphead;
+ int rc = getifaddrs(&ifaphead);
+ if (rc) return $no("cannot get network adapter list") false;
+
+ bool foundAdapter = false;
+ for (struct ifaddrs* ifap = ifaphead; ifap; ifap = ifap->ifa_next)
+ {
+ if (ifap->ifa_addr && ifap->ifa_addr->sa_family == AF_LINK)
+ {
+ struct sockaddr_dl* sdl = reinterpret_cast<struct sockaddr_dl*>(ifap->ifa_addr);
+ caddr_t ap = (caddr_t) (sdl->sdl_data + sdl->sdl_nlen);
+ int alen = sdl->sdl_alen;
+ if (ap && alen > 0)
+ {
+ _node.resize( alen );
+ std::memcpy(_node.data(), ap, _node.size() );
+ foundAdapter = true;
+ break;
+ }
+ }
+ }
+ freeifaddrs(ifaphead);
+ if (!foundAdapter) return $no("cannot determine MAC address (no suitable network adapter found)") false;
+ return true;
+ })
+
+ $osx({
+ struct ifaddrs* ifaphead;
+ int rc = getifaddrs(&ifaphead);
+ if (rc) return $no("cannot get network adapter list") false;
+
+ bool foundAdapter = false;
+ for (struct ifaddrs* ifap = ifaphead; ifap; ifap = ifap->ifa_next)
+ {
+ if (ifap->ifa_addr && ifap->ifa_addr->sa_family == AF_LINK)
+ {
+ struct sockaddr_dl* sdl = reinterpret_cast<struct sockaddr_dl*>(ifap->ifa_addr);
+ caddr_t ap = (caddr_t) (sdl->sdl_data + sdl->sdl_nlen);
+ int alen = sdl->sdl_alen;
+ if (ap && alen > 0)
+ {
+ _node.resize( alen );
+ std::memcpy(_node.data(), ap, _node.size() );
+ foundAdapter = true;
+ break;
+ }
+ }
+ }
+ freeifaddrs(ifaphead);
+ if (!foundAdapter) return $no("cannot determine MAC address (no suitable network adapter found)") false;
+ return true;
+ })
+
+ $linux({
+ struct ifreq ifr;
+
+ int s = socket(PF_INET, SOCK_DGRAM, 0);
+ if (s == -1) return $no("cannot open socket") false;
+
+ std::strcpy(ifr.ifr_name, "eth0");
+ int rc = ioctl(s, SIOCGIFHWADDR, &ifr);
+ close(s);
+ if (rc < 0) return $no("cannot get MAC address") false;
+ struct sockaddr* sa = reinterpret_cast<struct sockaddr*>(&ifr.ifr_addr);
+ _node.resize( sizeof(sa->sa_data) );
+ std::memcpy(_node.data(), sa->sa_data, _node.size() );
+ return true;
+ })
+
+ $unix({
+ char name[MAXHOSTNAMELEN];
+ if (gethostname(name, sizeof(name)))
+ return $no("cannot get host name") false;
+
+ struct hostent* pHost = gethostbyname(name);
+ if (!pHost) return $no("cannot get host IP address") false;
+
+ int s = socket(PF_INET, SOCK_DGRAM, IPPROTO_UDP);
+ if (s == -1) return $no("cannot open socket") false;
+
+ struct arpreq ar;
+ std::memset(&ar, 0, sizeof(ar));
+ struct sockaddr_in* pAddr = reinterpret_cast<struct sockaddr_in*>(&ar.arp_pa);
+ pAddr->sin_family = AF_INET;
+ std::memcpy(&pAddr->sin_addr, *pHost->h_addr_list, sizeof(struct in_addr));
+ int rc = ioctl(s, SIOCGARP, &ar);
+ close(s);
+ if (rc < 0) return $no("cannot get MAC address") false;
+ _node.resize( sizeof(ar.arp_ha.sa_data) );
+ std::memcpy(_node.data(), ar.arp_ha.sa_data, _node.size());
+ return true;
+ })
+ }
+
+ // Looks for first MAC address of any network device, size truncated to 48bits.
+ inline uint64_t get_any_mac48() {
+ std::vector<unsigned char> node;
+ if( get_any_mac(node) ) {
+ std::stringstream ss;
+ ss << std::hex << std::setfill('0');
+ node.resize(6);
+ for( unsigned i = 0; i < 6; ++i )
+ ss << std::setw(2) << int(node[i]);
+ uint64_t t;
+ if( ss >> t )
+ return t;
+ }
+ return 0;
+ }
+
+ //////////////////////////////////////////////////////////////////////////////////////
+ // UUID implementations
+
+ inline uuid uuid4() {
+ std::random_device rd;
+ std::uniform_int_distribution<uint64_t> dist(0, (uint64_t)(~0));
+
+ uuid my;
+
+ my.ab = dist(rd);
+ my.cd = dist(rd);
+
+ my.ab = (my.ab & 0xFFFFFFFFFFFF0FFFULL) | 0x0000000000004000ULL;
+ my.cd = (my.cd & 0x3FFFFFFFFFFFFFFFULL) | 0x8000000000000000ULL;
+
+ return my;
+ }
+
+ inline uuid uuid1() {
+ // Number of 100-ns intervals since 00:00:00.00 15 October 1582; [ref] uuid.py
+ uint64_t ns100_intervals = get_time( 0x01b21dd213814000ULL );
+ uint16_t clock_seq = (uint16_t)( ns100_intervals & 0x3fff ); // 14-bits max
+ uint64_t mac = get_any_mac48(); // 48-bits max
+
+ uint32_t time_low = ns100_intervals & 0xffffffff;
+ uint16_t time_mid = (ns100_intervals >> 32) & 0xffff;
+ uint16_t time_hi_version = (ns100_intervals >> 48) & 0xfff;
+ uint8_t clock_seq_low = clock_seq & 0xff;
+ uint8_t clock_seq_hi_variant = (clock_seq >> 8) & 0x3f;
+
+ uuid u;
+ uint64_t &upper_ = u.ab;
+ uint64_t &lower_ = u.cd;
+
+ // Build the high 32 bytes
+ upper_ = (uint64_t) time_low << 32;
+ upper_ |= (uint64_t) time_mid << 16;
+ upper_ |= (uint64_t) time_hi_version;
+
+ // Build the low 32 bytes, using the clock sequence number
+ lower_ = (uint64_t) ((clock_seq_hi_variant << 8) | clock_seq_low) << 48;
+ lower_ |= mac;
+
+ // Set the variant to RFC 4122.
+ lower_ &= ~((uint64_t)0xc000 << 48);
+ lower_ |= (uint64_t)0x8000 << 48;
+
+ // Set the version number.
+ enum { version = 1 };
+ upper_ &= ~0xf000;
+ upper_ |= version << 12;
+
+ return u;
+ }
+
+ inline uuid uuid0() {
+ // Number of 100-ns intervals since Unix epoch time
+ uint64_t ns100_intervals = get_time( 0 );
+ uint64_t pid = $windows( _getpid() ) $welse( getpid() );
+ uint16_t pid16 = (uint16_t)( pid & 0xffff ); // 16-bits max
+ uint64_t mac = get_any_mac48(); // 48-bits max
+
+ uint32_t time_low = ns100_intervals & 0xffffffff;
+ uint16_t time_mid = (ns100_intervals >> 32) & 0xffff;
+ uint16_t time_hi_version = (ns100_intervals >> 48) & 0xfff;
+ uint8_t pid_low = pid16 & 0xff;
+ uint8_t pid_hi = (pid16 >> 8) & 0xff;
+
+ uuid u;
+ uint64_t &upper_ = u.ab;
+ uint64_t &lower_ = u.cd;
+
+ // Build the high 32 bytes.
+ upper_ = (uint64_t) time_low << 32;
+ upper_ |= (uint64_t) time_mid << 16;
+ upper_ |= (uint64_t) time_hi_version;
+
+ // Build the low 32 bytes, using the mac and pid number.
+ lower_ = (uint64_t) ((pid_hi << 8) | pid_low) << 48;
+ lower_ |= mac;
+
+ // Set the version number.
+ enum { version = 0 };
+ upper_ &= ~0xf000;
+ upper_ |= version << 12;
+
+ return u;
+ }
+
+ inline uuid rebuild( uint64_t ab, uint64_t cd ) {
+ uuid u;
+ u.ab = ab, u.cd = cd;
+ return u;
+ }
+
+ inline uuid rebuild( const std::string &uustr ) {
+ char sep;
+ uint64_t a,b,c,d,e;
+ uuid u = { 0, 0 };
+ auto idx = uustr.find_first_of("-");
+ if( idx != std::string::npos ) {
+ const char *base62 =
+ "0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+ "abcdefghijklmnopqrstuvwxyz";
+ // single separator, base62 notation
+ if( uustr.find_first_of("-",idx+1) == std::string::npos ) {
+ u.ab = rebase( uustr.substr(0,idx), base62 );
+ u.cd = rebase( uustr.substr(idx+1), base62 );
+ }
+ // else classic hex notation
+ else {
+ std::stringstream ss( uustr );
+ if( ss >> std::hex >> a >> sep >> b >> sep >> c >> sep >> d >> sep >> e ) {
+ u.ab = (a << 32) | (b << 16) | c;
+ u.cd = (d << 48) | e;
+ }
+ }
+ }
+ return u;
+ }
+
+} // ::sole
+
+#undef $bsd
+#undef $belse
+#undef $linux
+#undef $lelse
+#undef $osx
+#undef $oelse
+#undef $unix
+#undef $uelse
+#undef $windows
+#undef $welse
+#undef $yes
+#undef $no
+
+// Pop disabled warnings
+#if (defined(__APPLE__) && defined(__MACH__))
+#pragma clang diagnostic pop
+#endif
\ No newline at end of file
diff --git a/include/store.h b/include/store.h
new file mode 100644
index 00000000..83f765c8
--- /dev/null
+++ b/include/store.h
@@ -0,0 +1,51 @@
+#pragma once
+
+#include <stdint.h>
+#include <string>
+#include <rocksdb/db.h>
+#include <rocksdb/options.h>
+
+/*
+ * Stores all information about a collection.
+ * Uses RocksDB for persistence.
+ */
+class Store {
+
+private:
+
+ std::string state_dir_path;
+
+ rocksdb::DB *db;
+ rocksdb::Options options;
+
+public:
+
+ Store() = delete;
+
+ Store(std::string state_dir_path): state_dir_path(state_dir_path) {
+ // Optimize RocksDB. This is the easiest way to get RocksDB to perform well
+ options.IncreaseParallelism();
+ options.OptimizeLevelStyleCompaction();
+ // create the DB if it's not already present
+ options.create_if_missing = true;
+
+ // open DB
+ rocksdb::Status s = rocksdb::DB::Open(options, state_dir_path, &db);
+ assert(s.ok());
+ }
+
+ ~Store() {
+ delete db;
+ }
+
+ bool insert(std::string&& key, std::string&& value) {
+ rocksdb::Status status = db->Put(rocksdb::WriteOptions(), key, value);
+ return status.ok();
+ }
+
+ std::string get(std::string&& key) {
+ std::string value;
+ rocksdb::Status status = db->Get(rocksdb::ReadOptions(), key, &value);
+ return value;
+ }
+};
\ No newline at end of file
diff --git a/src/collection.cpp b/src/collection.cpp
index cac2aedc..fbead280 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -6,24 +6,40 @@
#include <intersection.h>
#include <match_score.h>
#include <string_utils.h>
+#include "sole.hpp"
+#include "json.hpp"
-Collection::Collection() {
- state = CollectionState();
+Collection::Collection(std::string state_dir_path) {
+ store = new Store(state_dir_path);
art_tree_init(&t);
}
Collection::~Collection() {
+ delete store;
art_tree_destroy(&t);
}
-void Collection::add(nlohmann::json document) {
- uint32_t doc_id = state.nextId();
+uint32_t Collection::next_seq_id() {
+ return ++seq_id;
+}
+
+void Collection::add(std::string json_str) {
+ nlohmann::json document = nlohmann::json::parse(json_str);
+
+ if(document.count("id") == 0) {
+ sole::uuid u1 = sole::uuid1();
+ document["id"] = u1.base62();
+ }
+
+ uint32_t seq_id = next_seq_id();
+
+ store->insert(std::to_string(seq_id), document.dump());
+ store->insert(document["id"], std::to_string(seq_id));
- uint16_t score = document["points"];
std::vector<std::string> tokens;
StringUtils::tokenize(document["title"], tokens, " ", true);
- std::unordered_map<std::string, std::vector<uint32_t>> token_to_offsets;
+ std::unordered_map<std::string, std::vector<uint32_t>> token_to_offsets;
for(uint32_t i=0; i<tokens.size(); i++) {
auto token = tokens[i];
std::transform(token.begin(), token.end(), token.begin(), ::tolower);
@@ -32,8 +48,8 @@ void Collection::add(nlohmann::json document) {
for(auto & kv: token_to_offsets) {
art_document art_doc;
- art_doc.id = doc_id;
- art_doc.score = score;
+ art_doc.id = seq_id;
+ art_doc.score = document["points"];
art_doc.offsets_len = (uint32_t) kv.second.size();
art_doc.offsets = new uint32_t[kv.second.size()];
@@ -57,10 +73,9 @@ void Collection::add(nlohmann::json document) {
delete art_doc.offsets;
}
- doc_scores[doc_id] = score;
+ doc_scores[seq_id] = document["points"];
}
-
/*
1. Split the query into tokens
2. For each token, look up ids using exact lookup
@@ -70,15 +85,15 @@ void Collection::add(nlohmann::json document) {
4. Intersect the lists to find docs that match each phrase
5. Sort the docs based on some ranking criteria
*/
-void Collection::search(std::string query, size_t max_results) {
+std::vector<nlohmann::json> Collection::search(std::string query, size_t max_results) {
std::vector<std::string> tokens;
StringUtils::tokenize(query, tokens, " ", true);
std::vector<std::vector<art_leaf*>> token_leaves;
for(std::string token: tokens) {
std::vector<art_leaf*> leaves;
- int max_cost = 2;
- art_iter_fuzzy_prefix(&t, (const unsigned char *) token.c_str(), (int) token.length(), max_cost, 10, leaves);
+ int max_cost = 0;
+ art_iter_fuzzy_prefix(&t, (const unsigned char *) token.c_str(), (int) token.length()+1, max_cost, 10, leaves);
if(!leaves.empty()) {
for(auto i=0; i<leaves.size(); i++) {
//printf("%s - ", token.c_str());
@@ -90,7 +105,7 @@ void Collection::search(std::string query, size_t max_results) {
}
if(token_leaves.size() == 0) {
- return ;
+ return std::vector<nlohmann::json>();
}
//std::cout << "token_leaves.size = " << token_leaves.size() << std::endl;
@@ -132,10 +147,18 @@ void Collection::search(std::string query, size_t max_results) {
topster.sort();
+ std::vector<nlohmann::json> results;
+
for(uint32_t i=0; i<topster.size; i++) {
uint32_t id = topster.getKeyAt(i);
std::cout << "ID: " << id << std::endl;
+
+ const std::string value = store->get(std::to_string(id));
+ nlohmann::json document = nlohmann::json::parse(value);
+ results.push_back(document);
}
+
+ return results;
}
void Collection::score_results(Topster<100> &topster, const std::vector<art_leaf *> &query_suggestion,
diff --git a/src/collection.h b/src/collection.h
index 4f34e43f..7e0ef291 100644
--- a/src/collection.h
+++ b/src/collection.h
@@ -4,20 +4,28 @@
#include <vector>
#include <art.h>
#include <unordered_map>
-#include <collection_state.h>
+#include <store.h>
#include <topster.h>
#include <json.hpp>
class Collection {
private:
- CollectionState state;
+ Store* store;
+
+ // Integer ID used internally for bitmaps - not exposed to the client
+ uint32_t seq_id;
+
art_tree t;
std::unordered_map<uint32_t, uint16_t> doc_scores;
+
+ uint32_t next_seq_id();
+
public:
- Collection();
+ Collection() = delete;
+ Collection(std::string state_dir_path);
~Collection();
- void add(nlohmann::json document);
- void search(std::string query, size_t max_results);
+ void add(std::string json_str);
+ std::vector<nlohmann::json> search(std::string query, size_t max_results);
static inline std::vector<art_leaf *> _next_suggestion(const std::vector<std::vector<art_leaf *>> &token_leaves,
long long int n);
diff --git a/src/main.cpp b/src/main.cpp
index 822cb1f2..71ff72fc 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -7,21 +7,19 @@
#include <unordered_map>
#include "string_utils.h"
#include "collection.h"
-#include "json.hpp"
using namespace std;
int main() {
- Collection *collection = new Collection();
+ Collection *collection = new Collection("/tmp/typesense-data");
std::ifstream infile("/Users/kishore/others/wreally/typesense/test/documents.jsonl");
//std::ifstream infile("/Users/kishore/Downloads/hnstories.jsonl");
- std::string jsonline;
+ std::string json_line;
- while (std::getline(infile, jsonline)) {
- nlohmann::json document = nlohmann::json::parse(jsonline);
- collection->add(document);
+ while (std::getline(infile, json_line)) {
+ collection->add(json_line);
}
infile.close();
diff --git a/src/server.cpp b/src/server.cpp
index f1dea67e..68ec72de 100644
--- a/src/server.cpp
+++ b/src/server.cpp
@@ -16,7 +16,6 @@
#include <regex>
#include "string_utils.h"
#include "collection.h"
-#include "json.hpp"
#include "h2o.h"
#include "h2o/http1.h"
@@ -26,8 +25,7 @@
static h2o_globalconf_t config;
static h2o_context_t ctx;
static h2o_accept_ctx_t accept_ctx;
-static Collection *collection = new Collection();
-
+static Collection *collection = new Collection("/tmp/typesense-data");
static h2o_pathconf_t *register_handler(h2o_hostconf_t *hostconf, const char *path,
int (*on_req)(h2o_handler_t *, h2o_req_t *)) {
@@ -81,17 +79,26 @@ static int chunked_test(h2o_handler_t *self, h2o_req_t *req) {
printf("Query: %s\n", query_map["q"].c_str());
auto begin = std::chrono::high_resolution_clock::now();
- collection->search(query_map["q"], 100);
- long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();
- std::cout << "Time taken: " << timeMillis << "us" << std::endl;
- h2o_iovec_t body = h2o_strdup(&req->pool, "hello world\n", SIZE_MAX);
+ std::vector<nlohmann::json> results = collection->search(query_map["q"], 100);
+ nlohmann::json json_array = nlohmann::json::array();
+ for(nlohmann::json& result: results) {
+ json_array.push_back(result);
+ }
+
+ std::string json_str = json_array.dump();
+
+ std::cout << "JSON:" << json_str << std::endl;
+
+ h2o_iovec_t body = h2o_strdup(&req->pool, json_str.c_str(), SIZE_MAX);
req->res.status = 200;
req->res.reason = "OK";
h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CONTENT_TYPE, H2O_STRLIT("text/plain"));
h2o_start_response(req, &generator);
h2o_send(req, &body, 1, 1);
+ long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();
+ std::cout << "Time taken: " << timeMillis << "us" << std::endl;
return 0;
}
@@ -148,11 +155,10 @@ void index_documents() {
//std::ifstream infile("/Users/kishore/others/wreally/typesense/test/documents.jsonl");
std::ifstream infile("/Users/kishore/Downloads/hnstories.jsonl");
- std::string jsonline;
+ std::string json_line;
- while (std::getline(infile, jsonline)) {
- nlohmann::json document = nlohmann::json::parse(jsonline);
- collection->add(document);
+ while (std::getline(infile, json_line)) {
+ collection->add(json_line);
}
infile.close();