Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Ensure that topster is fully stable on equal values.

Kishore Nallan21c31de3b8d6
diff --git a/include/topster.h b/include/topster.h
index 4accd374..5d43920d 100644
--- a/include/topster.h
+++ b/include/topster.h
@@ -275,8 +275,8 @@ struct Topster {
}
static bool is_smaller(const struct KV* i, const struct KV* j) {
- return std::tie(i->scores[0], i->scores[1], i->scores[2]) <
- std::tie(j->scores[0], j->scores[1], j->scores[2]);
+ return std::tie(i->scores[0], i->scores[1], i->scores[2], i->key) <
+ std::tie(j->scores[0], j->scores[1], j->scores[2], j->key);
}
static bool is_greater_kv_group(const std::vector<KV*>& i, const std::vector<KV*>& j) {
diff --git a/test/resources/record_values.txt b/test/resources/record_values.txt
new file mode 100644
index 00000000..a512254f
--- /dev/null
+++ b/test/resources/record_values.txt
@@ -0,0 +1,816 @@
+316,-1200
+1735,-1200
+216,-1130
+1287,-1200
+2356,-1200
+590,-1200
+2079,-1200
+1843,-1130
+2467,-1200
+593,-1000
+326,-1130
+259,-1130
+2430,-1200
+1899,-1000
+717,-1030
+984,-1145
+1929,-1000
+2189,-1015
+2203,-1200
+2505,-1200
+1162,-1130
+1997,-1000
+2238,-1200
+1734,-1200
+686,-1130
+1829,-1200
+2181,-1200
+2185,-1200
+2038,-1000
+2366,-1200
+1920,-1030
+822,-1200
+407,-1100
+602,-1100
+767,-1100
+774,-1115
+630,-1000
+2176,-1200
+1726,-1030
+1702,-1130
+76,-1130
+93,-1130
+2633,-1130
+388,-1200
+715,-1145
+1306,-1100
+2451,-1030
+1231,-1000
+2288,-1200
+517,-1200
+501,-1000
+1263,-1130
+9,-1200
+465,-1130
+760,-1130
+165,-1200
+1667,-1200
+1870,-1130
+1819,-1200
+1130,-1100
+1249,-1200
+1428,-1100
+2255,-1200
+2407,-1000
+719,-1200
+558,-1200
+1525,-1130
+1662,-1000
+938,-1100
+826,-1130
+1803,-1200
+2426,-1030
+2018,-1130
+2471,-1000
+860,-1130
+2117,-1100
+1837,-1200
+114,-1100
+2556,-1200
+1023,-1100
+1326,-1130
+198,-1130
+716,-1100
+1725,-1200
+1668,-1200
+1484,-1000
+1445,-1145
+1649,-1200
+2547,-1100
+1092,-1030
+416,-1200
+1961,-1200
+762,-1030
+1230,-1200
+143,-1200
+340,-1200
+1557,-1200
+2300,-1130
+1177,-1130
+2370,-1130
+1253,-1130
+2368,-1200
+1279,-1100
+794,-1200
+1323,-1130
+277,-1145
+1309,-1200
+2250,-1200
+2594,-1200
+1848,-1130
+1952,-1000
+2199,-1100
+2134,-1200
+321,-1100
+472,-1200
+591,-1200
+605,-1200
+62,-1200
+1912,-1145
+2511,-1200
+1305,-1200
+1695,-1100
+567,-1130
+1430,-1015
+1665,-1200
+265,-1130
+487,-1130
+2375,-1000
+1745,-1030
+2144,-1145
+684,-1145
+1224,-1200
+1914,-1200
+110,-1200
+525,-1030
+1689,-1100
+549,-1130
+2461,-1130
+2416,-1130
+65,-1100
+923,-1130
+1334,-1200
+1915,-1200
+1859,-1130
+504,-1145
+1767,-1200
+1589,-1030
+2361,-1130
+579,-1100
+2399,-1130
+1173,-1200
+1282,-1000
+2228,-1000
+1805,-1000
+601,-1200
+111,-1200
+1810,-1015
+2215,-1200
+917,-1145
+1838,-1030
+2401,-1100
+2546,-1200
+570,-1100
+2195,-1200
+393,-1100
+2480,-1200
+1882,-1130
+1328,-1200
+2128,-1045
+1302,-1100
+494,-1100
+2171,-1200
+2141,-1115
+1847,-1145
+825,-1200
+980,-1045
+2393,-1130
+1799,-1030
+2395,-1030
+2630,-1130
+2209,-1200
+1004,-1000
+1916,-1030
+2048,-1030
+299,-1130
+1913,-1200
+1854,-1000
+1943,-1200
+2071,-1200
+1770,-1200
+729,-1145
+802,-1200
+2294,-1200
+2161,-1015
+965,-1100
+2239,-1200
+642,-1130
+1985,-1000
+1750,-1100
+1842,-1200
+1785,-1130
+2434,-1200
+1314,-1100
+759,-1200
+547,-1130
+973,-1130
+1320,-1130
+1174,-1100
+952,-1000
+95,-1200
+1650,-1100
+443,-1200
+2436,-1200
+1889,-1000
+2580,-1200
+2481,-1130
+31,-1200
+808,-1200
+688,-1000
+2220,-1115
+1704,-1000
+2073,-1000
+1225,-1100
+2415,-1200
+983,-1200
+598,-1200
+2491,-1130
+402,-1000
+699,-1200
+967,-1000
+728,-1200
+426,-1030
+492,-1130
+158,-1200
+709,-1100
+1164,-1200
+1753,-1130
+1594,-1000
+1374,-1200
+1741,-1030
+2324,-1030
+2563,-1000
+2445,-1200
+2468,-1200
+2053,-1100
+1593,-1200
+1400,-1030
+1436,-1200
+2406,-1200
+2113,-1145
+950,-1130
+1434,-1130
+1212,-1100
+693,-1200
+899,-1030
+1014,-1200
+1911,-1200
+2116,-1200
+2097,-1030
+1031,-1200
+1944,-1200
+1298,-1200
+909,-1145
+2542,-1015
+10,-1200
+848,-1100
+2557,-1130
+1959,-1130
+2386,-1200
+392,-1145
+2408,-1130
+2474,-1100
+1776,-1200
+1755,-1030
+2223,-1130
+2172,-1030
+2196,-1100
+1012,-1115
+569,-1100
+2216,-1030
+2030,-1130
+1192,-1000
+2101,-1200
+1228,-1030
+1771,-1130
+1876,-1200
+512,-1100
+2150,-1000
+2056,-1100
+1437,-1145
+1679,-1200
+1515,-1045
+177,-1000
+511,-1200
+2193,-1200
+2579,-1200
+72,-1130
+2489,-1200
+163,-1200
+2486,-1115
+218,-1100
+1924,-1000
+1678,-1200
+702,-1200
+1783,-1130
+1075,-1130
+1825,-1015
+1375,-1030
+778,-1100
+793,-1145
+2410,-1130
+1297,-1200
+1373,-1130
+2628,-1200
+116,-1100
+1439,-1000
+1957,-1130
+2626,-1130
+2008,-1200
+2064,-1000
+987,-1200
+506,-1130
+1964,-1130
+2458,-1130
+2446,-1000
+318,-1200
+1026,-1200
+1329,-1030
+1137,-1130
+2031,-1000
+448,-1130
+1559,-1130
+1950,-1000
+1264,-1100
+2599,-1130
+164,-1200
+656,-1145
+930,-1200
+592,-1200
+1165,-1000
+2490,-1200
+2135,-1100
+273,-1130
+2160,-1200
+381,-1100
+2240,-1130
+384,-1000
+1768,-1130
+172,-1200
+766,-1200
+1015,-1200
+2433,-1030
+106,-1200
+1269,-1130
+1680,-1000
+148,-1200
+1496,-1130
+2226,-1130
+2282,-1200
+1709,-1100
+430,-1200
+2165,-1200
+1271,-1200
+2443,-1130
+1027,-1200
+1038,-1130
+859,-1115
+2577,-1130
+2369,-1200
+857,-1130
+1140,-1200
+636,-1130
+1642,-1030
+1660,-1200
+2212,-1130
+1535,-1200
+941,-1200
+1795,-1030
+919,-1130
+720,-1100
+1688,-1200
+1562,-1100
+297,-1130
+89,-1200
+1909,-1130
+864,-1200
+2127,-1200
+64,-1200
+1024,-1200
+1701,-1100
+2202,-1200
+2251,-1130
+1019,-1100
+2629,-1200
+931,-1200
+1123,-1000
+1804,-1200
+635,-1200
+2532,-1045
+752,-1100
+1973,-1000
+2023,-1000
+782,-1000
+166,-1000
+998,-1200
+200,-1000
+1510,-1000
+1937,-1100
+697,-1130
+63,-1200
+576,-1200
+2585,-1030
+1732,-1130
+537,-1000
+2136,-1000
+36,-1200
+2133,-1200
+2529,-1130
+1491,-1200
+2241,-1000
+258,-1200
+856,-1145
+2344,-1000
+1063,-1200
+1782,-1200
+149,-1200
+1073,-1130
+239,-1130
+2477,-1100
+2232,-1130
+187,-1115
+499,-1200
+1142,-1100
+433,-1200
+2598,-1145
+1340,-1000
+1256,-1000
+1670,-1030
+505,-1200
+1028,-1100
+152,-1130
+1528,-1100
+2167,-1200
+519,-1000
+1184,-1100
+2550,-1200
+1763,-1100
+921,-1200
+2329,-1200
+445,-1145
+2036,-1100
+317,-1200
+1527,-1145
+963,-1130
+2413,-1130
+798,-1100
+2611,-1015
+1877,-1200
+1034,-1130
+2332,-1000
+2363,-1200
+1009,-1130
+685,-1200
+1251,-1130
+1055,-1130
+2431,-1130
+2178,-1100
+661,-1200
+700,-1030
+1817,-1130
+777,-1145
+1186,-1130
+1655,-1100
+2601,-1045
+1835,-1130
+1567,-1000
+2365,-1200
+2103,-1030
+279,-1015
+2291,-1200
+583,-1100
+2418,-1130
+2177,-1200
+554,-1200
+1687,-1200
+795,-1200
+1189,-1100
+2276,-1000
+670,-1130
+1506,-1100
+390,-1200
+573,-1200
+2496,-1000
+1723,-1130
+1139,-1200
+2501,-1130
+740,-1030
+37,-1200
+1749,-1200
+210,-1130
+1335,-1000
+1504,-1130
+2131,-1130
+2086,-1200
+813,-1200
+2521,-1000
+2184,-1200
+694,-1100
+1170,-1045
+975,-1030
+796,-1130
+1157,-1000
+2562,-1200
+171,-1145
+2210,-1130
+542,-1200
+1561,-1200
+2330,-1200
+2085,-1200
+574,-1130
+2589,-1200
+205,-1200
+1814,-1100
+1188,-1200
+2516,-1000
+156,-1200
+2535,-1130
+1333,-1200
+162,-1200
+2636,-1145
+1820,-1000
+444,-1200
+1599,-1100
+150,-1200
+1077,-1130
+2371,-1030
+1637,-1000
+1698,-1100
+452,-1200
+1872,-1015
+2378,-1130
+2080,-1000
+128,-1130
+587,-1100
+1368,-1000
+804,-1000
+1766,-1200
+1940,-1100
+2605,-1145
+1558,-1200
+2627,-1200
+1532,-1130
+1272,-1130
+2359,-1200
+2331,-1200
+1509,-1200
+2047,-1200
+2495,-1200
+993,-1000
+1759,-1030
+2380,-1200
+585,-1030
+1711,-1200
+1661,-1200
+2469,-1130
+1574,-1200
+2435,-1200
+1945,-1000
+104,-1130
+707,-1200
+410,-1200
+979,-1200
+1518,-1200
+453,-1200
+1083,-1030
+988,-1000
+1941,-1200
+2595,-1130
+1551,-1000
+1521,-1030
+1866,-1200
+2188,-1200
+1274,-1000
+413,-1130
+2281,-1200
+1790,-1100
+391,-1200
+1852,-1030
+1057,-1130
+2603,-1130
+1575,-1000
+497,-1130
+1867,-1200
+600,-1200
+915,-1130
+725,-1100
+1894,-1000
+467,-1000
+1572,-1130
+1236,-1030
+1693,-1130
+2364,-1200
+1685,-1200
+820,-1130
+947,-1200
+942,-1000
+1737,-1030
+2109,-1030
+2509,-1200
+255,-1100
+1777,-1000
+2421,-1000
+12,-1200
+2320,-1200
+1878,-1130
+2488,-1200
+2072,-1200
+81,-1100
+1215,-1130
+2253,-1130
+136,-1200
+2214,-1200
+1288,-1000
+2120,-1115
+1081,-1130
+2234,-1130
+301,-1130
+2420,-1200
+1658,-1200
+1229,-1200
+1677,-1200
+530,-1115
+2295,-1200
+758,-1200
+925,-1000
+1659,-1200
+2510,-1200
+2002,-1000
+500,-1200
+751,-1200
+2411,-1130
+908,-1200
+2462,-1000
+1250,-1200
+518,-1200
+2485,-1200
+2315,-1000
+1322,-1200
+2107,-1130
+2367,-1200
+1730,-1130
+242,-1200
+599,-1200
+801,-1145
+582,-1200
+1128,-1130
+338,-1030
+1498,-1030
+1217,-1000
+2194,-1200
+2559,-1100
+509,-1130
+2537,-1000
+2506,-1200
+2108,-1145
+1978,-1030
+2615,-1100
+1519,-1000
+1861,-1000
+1444,-1200
+1736,-1200
+1011,-1145
+809,-1200
+2317,-1100
+489,-1200
+1267,-1145
+2341,-1200
+2078,-1200
+905,-1115
+2028,-1030
+485,-1130
+2387,-1200
+949,-1200
+2404,-1200
+411,-1200
+2405,-1200
+270,-1100
+1040,-1100
+2087,-1200
+907,-1200
+1067,-1100
+1968,-1000
+985,-1145
+53,-1200
+1963,-1200
+1492,-1030
+275,-1130
+2309,-1200
+824,-1200
+2021,-1130
+2531,-1200
+703,-1030
+1112,-1000
+1845,-1030
+1669,-1200
+972,-1200
+2340,-1200
+1773,-1100
+2143,-1200
+333,-1130
+415,-1200
+137,-1000
+851,-1000
+1962,-1200
+2623,-1100
+2507,-1200
+319,-1130
+305,-1130
+2321,-1100
+1686,-1200
+2307,-1130
+1456,-1200
+2155,-1000
+875,-1200
+2236,-1145
+151,-1200
+555,-1115
+2381,-1000
+113,-1200
+2343,-1200
+755,-1200
+1062,-1200
+2225,-1200
+1904,-1000
+1712,-1000
+2102,-1200
+803,-1200
+1451,-1000
+1674,-1100
+2493,-1130
+1692,-1200
+2360,-1200
+2310,-1000
+1090,-1030
+2052,-1200
+2349,-1200
+389,-1200
+1580,-1200
+978,-1200
+524,-1200
+1070,-1200
+2618,-1145
+1037,-1200
+922,-1130
+1935,-1130
+823,-1200
+449,-1100
+1223,-1200
+2526,-1200
+2292,-1100
+1565,-1130
+708,-1200
+2222,-1200
+1666,-1200
+2572,-1000
+335,-1130
+1880,-1130
+2122,-1000
+1072,-1200
+1966,-1130
+1457,-1100
+948,-1200
+781,-1200
+304,-1200
+1934,-1115
+516,-1200
+2,-1100
+1240,-1000
+2204,-1200
+1868,-1130
+1099,-1200
+370,-1000
+787,-1015
+1830,-1130
+578,-1145
+32,-1200
+1531,-1200
+1255,-1145
+2551,-1000
+831,-1000
+1102,-1000
+2619,-1030
+1646,-1100
+1832,-1200
+412,-1200
+1502,-1130
+80,-1200
+379,-1130
+387,-1200
+261,-1130
+1581,-1030
+2342,-1200
+55,-1200
+3,-1145
+1036,-1200
+2182,-1130
+1059,-1130
+657,-1100
+1022,-1200
+1884,-1000
+2007,-1200
+57,-1200
+712,-1000
+90,-1200
+529,-1200
+1025,-1200
\ No newline at end of file
diff --git a/test/topster_test.cpp b/test/topster_test.cpp
index 304b5a8a..858448c3 100644
--- a/test/topster_test.cpp
+++ b/test/topster_test.cpp
@@ -2,6 +2,7 @@
#include <index.h>
#include "topster.h"
#include "match_score.h"
+#include <fstream>
TEST(TopsterTest, MaxIntValues) {
Topster topster(5);
@@ -58,6 +59,84 @@ TEST(TopsterTest, MaxIntValues) {
}
}
+TEST(TopsterTest, StableSorting) {
+ // evaluate if the positions of the documents in Topster<1000> is the same in Topster 250, 500 and 750
+
+ std::ifstream infile(std::string(ROOT_DIR)+"test/resources/record_values.txt");
+ std::string line;
+ std::vector<std::pair<uint64_t, int64_t>> records;
+
+ while (std::getline(infile, line)) {
+ std::vector<std::string> parts;
+ StringUtils::split(line, parts, ",");
+ uint64_t key = std::stoll(parts[0]);
+ records.emplace_back(key, std::stoi(parts[1]));
+ }
+
+ infile.close();
+
+ Topster topster1K(1000);
+
+ for(auto id_score: records) {
+ int64_t scores[3] = {id_score.second, 0, 0};
+ KV kv(0, 0, 0, id_score.first, id_score.first, 0, scores);
+ topster1K.add(&kv);
+ }
+
+ topster1K.sort();
+
+ std::vector<uint64_t> record_ids;
+
+ for(uint32_t i = 0; i < topster1K.size; i++) {
+ record_ids.push_back(topster1K.getKeyAt(i));
+ }
+
+ // check on Topster<250>
+ Topster topster250(250);
+
+ for(auto id_score: records) {
+ int64_t scores[3] = {id_score.second, 0, 0};
+ KV kv(0, 0, 0, id_score.first, id_score.first, 0, scores);
+ topster250.add(&kv);
+ }
+
+ topster250.sort();
+
+ for(uint32_t i = 0; i < topster250.size; i++) {
+ ASSERT_EQ(record_ids[i], topster250.getKeyAt(i));
+ }
+
+ // check on Topster<500>
+ Topster topster500(500);
+
+ for(auto id_score: records) {
+ int64_t scores[3] = {id_score.second, 0, 0};
+ KV kv(0, 0, 0, id_score.first, id_score.first, 0, scores);
+ topster500.add(&kv);
+ }
+
+ topster500.sort();
+
+ for(uint32_t i = 0; i < topster500.size; i++) {
+ ASSERT_EQ(record_ids[i], topster500.getKeyAt(i));
+ }
+
+ // check on Topster<750>
+ Topster topster750(750);
+
+ for(auto id_score: records) {
+ int64_t scores[3] = {id_score.second, 0, 0};
+ KV kv(0, 0, 0, id_score.first, id_score.first, 0, scores);
+ topster750.add(&kv);
+ }
+
+ topster750.sort();
+
+ for(uint32_t i = 0; i < topster750.size; i++) {
+ ASSERT_EQ(record_ids[i], topster750.getKeyAt(i));
+ }
+}
+
TEST(TopsterTest, MaxFloatValues) {
Topster topster(5);