Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
JSON document as input to `collection.add` method.
Kishore Nallan1d3af330ddf2
diff --git a/TODO.md b/TODO.mdindex 7cb94a95..061f2fa9 100644--- a/TODO.md+++ b/TODO.md@@ -4,7 +4,7 @@**Search index**-- Proper JSON as input+- ~~Proper JSON as input~~- Storing raw JSON input to RocksDB- ART for every indexed field- UTF-8 support for fuzzy searchdiff --git a/src/collection.cpp b/src/collection.cppindex deda470d..cac2aedc 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -2,12 +2,10 @@#include <iostream>#include <numeric>-#include <unordered_map>#include <topster.h>#include <intersection.h>#include <match_score.h>#include <string_utils.h>-#include <art.h>Collection::Collection() {state = CollectionState();@@ -18,8 +16,12 @@ Collection::~Collection() {art_tree_destroy(&t);}-void Collection::add(std::vector<std::string> tokens, uint16_t score) {+void Collection::add(nlohmann::json document) {uint32_t doc_id = state.nextId();++ uint16_t score = document["points"];+ std::vector<std::string> tokens;+ StringUtils::tokenize(document["title"], tokens, " ", true);std::unordered_map<std::string, std::vector<uint32_t>> token_to_offsets;for(uint32_t i=0; i<tokens.size(); i++) {@@ -29,11 +31,11 @@ void Collection::add(std::vector<std::string> tokens, uint16_t score) {}for(auto & kv: token_to_offsets) {- art_document document;- document.id = doc_id;- document.score = score;- document.offsets_len = (uint32_t) kv.second.size();- document.offsets = new uint32_t[kv.second.size()];+ art_document art_doc;+ art_doc.id = doc_id;+ art_doc.score = score;+ art_doc.offsets_len = (uint32_t) kv.second.size();+ art_doc.offsets = new uint32_t[kv.second.size()];uint32_t num_hits = 0;@@ -48,11 +50,11 @@ void Collection::add(std::vector<std::string> tokens, uint16_t score) {num_hits += 1;for(auto i=0; i<kv.second.size(); i++) {- document.offsets[i] = kv.second[i];+ art_doc.offsets[i] = kv.second[i];}- art_insert(&t, (const unsigned char *) key, key_len, &document, num_hits);- delete document.offsets;+ art_insert(&t, key, key_len, &art_doc, num_hits);+ delete art_doc.offsets;}doc_scores[doc_id] = score;diff --git a/src/collection.h b/src/collection.hindex 16cba664..4f34e43f 100644--- a/src/collection.h+++ b/src/collection.h@@ -6,6 +6,7 @@#include <unordered_map>#include <collection_state.h>#include <topster.h>+#include <json.hpp>class Collection {private:@@ -15,7 +16,7 @@ private:public:Collection();~Collection();- void add(std::vector<std::string> tokens, uint16_t score);+ void add(nlohmann::json document);void search(std::string query, size_t max_results);static inline std::vector<art_leaf *> _next_suggestion(const std::vector<std::vector<art_leaf *>> &token_leaves,diff --git a/src/main.cpp b/src/main.cppindex 3eacb753..822cb1f2 100644--- a/src/main.cpp+++ b/src/main.cpp@@ -7,29 +7,24 @@#include <unordered_map>#include "string_utils.h"#include "collection.h"+#include "json.hpp"using namespace std;int main() {Collection *collection = new Collection();- std::ifstream infile("/Users/kishore/others/wreally/typesense/test/documents.txt");- //std::ifstream infile("/Users/kishore/Downloads/hnstories.tsv");+ std::ifstream infile("/Users/kishore/others/wreally/typesense/test/documents.jsonl");+ //std::ifstream infile("/Users/kishore/Downloads/hnstories.jsonl");- std::string line;+ std::string jsonline;- while (std::getline(infile, line)) {- vector<string> parts;- StringUtils::tokenize(line, parts, "\t", true);- line = StringUtils::replace_all(line, "\"", "");-- vector<string> tokens;- StringUtils::tokenize(parts[0], tokens, " ", true);-- if(parts.size() != 2) continue;- collection->add(tokens, stoi(parts[1]));+ while (std::getline(infile, jsonline)) {+ nlohmann::json document = nlohmann::json::parse(jsonline);+ collection->add(document);}+ infile.close();cout << "FINISHED INDEXING!" << endl << flush;auto begin = std::chrono::high_resolution_clock::now();diff --git a/src/server.cpp b/src/server.cppindex 3769b606..f1dea67e 100644--- a/src/server.cpp+++ b/src/server.cpp@@ -145,23 +145,17 @@ static int create_listener(void) {}void index_documents() {- //std::ifstream infile("/Users/kishore/others/wreally/typesense/test/documents.txt");- std::ifstream infile("/Users/kishore/Downloads/hnstories.tsv");+ //std::ifstream infile("/Users/kishore/others/wreally/typesense/test/documents.jsonl");+ std::ifstream infile("/Users/kishore/Downloads/hnstories.jsonl");- std::string line;+ std::string jsonline;- while (std::getline(infile, line)) {- std::vector<std::string> parts;- StringUtils::tokenize(line, parts, "\t", true);- line = StringUtils::replace_all(line, "\"", "");-- std::vector<std::string> tokens;- StringUtils::tokenize(parts[0], tokens, " ", true);-- if(parts.size() != 2) continue;- collection->add(tokens, stoi(parts[1]));+ while (std::getline(infile, jsonline)) {+ nlohmann::json document = nlohmann::json::parse(jsonline);+ collection->add(document);}+ infile.close();std::cout << "FINISHED INDEXING!" << std::endl << std::flush;}diff --git a/test/documents.jsonl b/test/documents.jsonlnew file mode 100644index 00000000..89c0c552--- /dev/null+++ b/test/documents.jsonl@@ -0,0 +1,16 @@+{"points":15,"title":"How are cryogenic rocket propellants delivered to the launch pad?"}+{"points":14,"title":"Are there any (free) are online data archives for data from instruments on Soviet / Russian missions?"}+{"points":13,"title":"Where should I look in ISS to find mouldy food?"}+{"points":13,"title":"Is solar system active cryovolcanism a potential viable power source for future colonies?"}+{"points":13,"title":"The heaviest martian spacecraft"}+{"points":13,"title":"To what extent are the US modules of ISS based on the Spacelab design?"}+{"points":12,"title":"Could future astronauts eat during EVAs?"}+{"points":12,"title":"What is the power requirement of a spacesuit?"}+{"points":12,"title":"How does plant growing medium not scatter around?"}+{"points":12,"title":"Is there research for the optimal small crew size for a long space voyage?"}+{"points":12,"title":"Do long term missions receive insurance coverage?"}+{"points":12,"title":"What do they exactly look for when searching for extraterrestrial intelligence?"}+{"points":11,"title":"What were emergency procedures for failure of launch vehicles with nuclear upper stages?"}+{"points":11,"title":"Mathematics used for F9R flyback lunch and landing"}+{"points":11,"title":"What considerations have been made lunch for waste produced during colonisation?"}+{"points":10,"title":"Do late do the propellants lunch ionize in chemical rockets?"}\ No newline at end of filediff --git a/test/documents.txt b/test/documents.txtdeleted file mode 100644index d276c2a6..00000000--- a/test/documents.txt+++ /dev/null@@ -1,16 +0,0 @@-How are cryogenic rocket propellants delivered to the launch pad? 15-Are there any (free) are online data archives for data from instruments on Soviet / Russian missions? 14-Where should I look in ISS to find mouldy food? 13-Is solar system active cryovolcanism a potential viable power source for future colonies? 13-The heaviest martian spacecraft 13-To what extent are the US modules of ISS based on the Spacelab design? 13-Could future astronauts eat during EVAs? 12-What is the power requirement of a spacesuit? 12-How does plant growing medium not scatter around? 12-Is there research for the optimal small crew size for a long space voyage? 12-Do long term missions receive insurance coverage? 12-What do they exactly look for when searching for extraterrestrial intelligence? 12-What were emergency procedures for failure of launch vehicles with nuclear upper stages? 11-Mathematics used for F9R flyback lunch and landing 11-What considerations have been made lunch for waste produced during colonisation? 11-Do late do the propellants lunch ionize in chemical rockets? 10\ No newline at end of file