web search done right
crossencoder ranking (finally!)
Mikkel Denker0a4650440217
diff --git a/Cargo.lock b/Cargo.lockindex a4f69b8d..28ebba62 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -327,6 +327,15 @@ dependencies = ["generic-array",]+[[package]]+name = "block-buffer"+version = "0.10.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e"+dependencies = [+ "generic-array",+]+[[package]]name = "bstr"version = "0.2.17"@@ -384,6 +393,29 @@ dependencies = ["pkg-config",]+[[package]]+name = "cached-path"+version = "0.5.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5f1c56d30236522ab3393a08746b138d4e16372001f42d29c88d513aeb8ab7ef"+dependencies = [+ "flate2",+ "fs2",+ "glob",+ "indicatif 0.16.2",+ "log",+ "rand 0.8.5",+ "reqwest",+ "serde",+ "serde_json",+ "sha2 0.10.6",+ "tar",+ "tempfile",+ "thiserror",+ "zip",+ "zip-extensions",+]+[[package]]name = "cast"version = "0.3.0"@@ -435,6 +467,12 @@ dependencies = ["winapi",]+[[package]]+name = "chunked_transfer"+version = "1.4.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cca491388666e04d7248af3f60f0c40cfb0991c72205595d7c396e3510207d1a"+[[package]]name = "clang-sys"version = "1.3.3"@@ -452,9 +490,13 @@ version = "2.34.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"dependencies = [+ "ansi_term",+ "atty","bitflags",+ "strsim 0.8.0","textwrap 0.11.0","unicode-width",+ "vec_map",][[package]]@@ -469,7 +511,7 @@ dependencies = ["clap_lex","indexmap","lazy_static",- "strsim",+ "strsim 0.10.0","termcolor","textwrap 0.15.0",]@@ -520,7 +562,9 @@ dependencies = ["encode_unicode","libc","once_cell",+ "regex","terminal_size",+ "unicode-width","winapi",]@@ -575,7 +619,7 @@ dependencies = ["clap 2.34.0","criterion-plot","csv",- "itertools",+ "itertools 0.10.3","lazy_static","num-traits","oorandom",@@ -597,7 +641,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "2673cc8207403546f45f5fd319a974b1e6983ad1a3ee7e6041650013be041876"dependencies = ["cast",- "itertools",+ "itertools 0.10.3",][[package]]@@ -651,6 +695,16 @@ version = "0.2.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"+[[package]]+name = "crypto-common"+version = "0.1.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"+dependencies = [+ "generic-array",+ "typenum",+]+[[package]]name = "crypto-mac"version = "0.11.1"@@ -736,8 +790,8 @@ dependencies = ["html-escape","http","image",- "indicatif",- "itertools",+ "indicatif 0.16.2",+ "itertools 0.10.3","kuchiki","logos","maplit",@@ -746,6 +800,7 @@ dependencies = ["min-max-heap","num_cpus","once_cell",+ "onnxruntime","optics","parse_wiki_text","quick-xml",@@ -762,6 +817,7 @@ dependencies = ["serde_urlencoded","tantivy","thiserror",+ "tokenizers","tokio","toml","tower-http",@@ -772,6 +828,41 @@ dependencies = ["whatlang",]+[[package]]+name = "darling"+version = "0.10.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0d706e75d87e35569db781a9b5e2416cff1236a47ed380831f959382ccd5f858"+dependencies = [+ "darling_core",+ "darling_macro",+]++[[package]]+name = "darling_core"+version = "0.10.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f0c960ae2da4de88a91b2d920c2a7233b400bc33cb28453a2987822d8392519b"+dependencies = [+ "fnv",+ "ident_case",+ "proc-macro2",+ "quote",+ "strsim 0.9.3",+ "syn",+]++[[package]]+name = "darling_macro"+version = "0.10.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d9b5a2f4ac4969822c62224815d069952656cadc7084fdca9751e6d959189b72"+dependencies = [+ "darling_core",+ "quote",+ "syn",+]+[[package]]name = "deflate"version = "1.0.0"@@ -781,6 +872,31 @@ dependencies = ["adler32",]+[[package]]+name = "derive_builder"+version = "0.9.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a2658621297f2cf68762a6f7dc0bb7e1ff2cfd6583daef8ee0fed6f7ec468ec0"+dependencies = [+ "darling",+ "derive_builder_core",+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "derive_builder_core"+version = "0.9.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2791ea3e372c8495c0bc2033991d76b512cd799d07491fbd6890124db9458bef"+dependencies = [+ "darling",+ "proc-macro2",+ "quote",+ "syn",+]+[[package]]name = "derive_more"version = "0.99.17"@@ -809,6 +925,25 @@ dependencies = ["generic-array",]+[[package]]+name = "digest"+version = "0.10.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f"+dependencies = [+ "block-buffer 0.10.3",+ "crypto-common",+]++[[package]]+name = "dirs"+version = "3.0.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "30baa043103c9d0c2a57cf537cc2f35623889dc0d405e6c3cccfadbc81c71309"+dependencies = [+ "dirs-sys",+]+[[package]]name = "dirs-next"version = "2.0.0"@@ -819,6 +954,17 @@ dependencies = ["dirs-sys-next",]+[[package]]+name = "dirs-sys"+version = "0.3.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6"+dependencies = [+ "libc",+ "redox_users",+ "winapi",+]+[[package]]name = "dirs-sys-next"version = "0.1.2"@@ -881,6 +1027,15 @@ dependencies = ["cfg-if",]+[[package]]+name = "esaxx-rs"+version = "0.1.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1f748b253ceca9fed5f42f8b5ceb3851e93102199bc25b64b65369f76e5c0a35"+dependencies = [+ "cc",+]+[[package]]name = "exr"version = "1.4.2"@@ -920,7 +1075,7 @@ version = "0.3.0"source = "git+https://github.com/quickwit-oss/tantivy.git?rev=5d4535de83d214765c29501f0418f1ff38a607b1#5d4535de83d214765c29501f0418f1ff38a607b1"dependencies = ["fastdivide",- "itertools",+ "itertools 0.10.3","log","ownedbytes","tantivy-bitpacker",@@ -936,6 +1091,18 @@ dependencies = ["instant",]+[[package]]+name = "filetime"+version = "0.2.19"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4e884668cd0c7480504233e951174ddc3b382f7c2666e3b7310b5c4e7b0c37f9"+dependencies = [+ "cfg-if",+ "libc",+ "redox_syscall",+ "windows-sys 0.42.0",+]+[[package]]name = "fixedbitset"version = "0.4.2"@@ -1272,7 +1439,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b"dependencies = ["crypto-mac",- "digest",+ "digest 0.9.0",][[package]]@@ -1402,6 +1569,12 @@ dependencies = ["tokio-native-tls",]+[[package]]+name = "ident_case"+version = "1.0.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"+[[package]]name = "idna"version = "0.2.3"@@ -1442,6 +1615,18 @@ dependencies = ["hashbrown 0.12.1",]+[[package]]+name = "indicatif"+version = "0.15.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7baab56125e25686df467fe470785512329883aab42696d661247aca2a2896e4"+dependencies = [+ "console",+ "lazy_static",+ "number_prefix 0.3.0",+ "regex",+]+[[package]]name = "indicatif"version = "0.16.2"@@ -1450,7 +1635,7 @@ checksum = "2d207dc617c7a380ab07ff572a6e52fa202a2a8f355860ac9c38e23f8196be1b"dependencies = ["console","lazy_static",- "number_prefix",+ "number_prefix 0.4.0","rayon","regex",]@@ -1482,6 +1667,24 @@ version = "2.5.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b"+[[package]]+name = "itertools"+version = "0.8.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f56a2d0bc861f9165be4eb3442afd3c236d8a98afd426f65d92324ae1091a484"+dependencies = [+ "either",+]++[[package]]+name = "itertools"+version = "0.9.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b"+dependencies = [+ "either",+]+[[package]]name = "itertools"version = "0.10.3"@@ -1553,7 +1756,7 @@ dependencies = ["bit-set","diff","ena",- "itertools",+ "itertools 0.10.3","lalrpop-util","petgraph","pico-args",@@ -1717,6 +1920,22 @@ version = "0.1.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"+[[package]]+name = "macro_rules_attribute"+version = "0.1.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cf0c9b980bf4f3a37fd7b1c066941dd1b1d0152ce6ee6e8fe8c49b9f6810d862"+dependencies = [+ "macro_rules_attribute-proc_macro",+ "paste",+]++[[package]]+name = "macro_rules_attribute-proc_macro"+version = "0.1.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "58093314a45e00c77d5c508f76e77c3396afbbc0d01506e7fae47b018bac2b1d"+[[package]]name = "maplit"version = "1.0.2"@@ -1758,14 +1977,23 @@ version = "0.5.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb"+[[package]]+name = "matrixmultiply"+version = "0.3.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "add85d4dd35074e6fedc608f8c8f513a3548619a9024b751949ef0e8e45a4d84"+dependencies = [+ "rawpointer",+]+[[package]]name = "md-5"version = "0.9.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15"dependencies = [- "block-buffer",- "digest",+ "block-buffer 0.9.0",+ "digest 0.9.0","opaque-debug",]@@ -1865,7 +2093,7 @@ dependencies = ["libc","log","wasi 0.11.0+wasi-snapshot-preview1",- "windows-sys",+ "windows-sys 0.36.1",][[package]]@@ -1904,6 +2132,19 @@ dependencies = ["tempfile",]+[[package]]+name = "ndarray"+version = "0.15.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cc1372704f14bb132a49a6701c2238970a359ee0829fed481b522a63bf25456a"+dependencies = [+ "matrixmultiply",+ "num-complex",+ "num-integer",+ "num-traits",+ "rawpointer",+]+[[package]]name = "new_debug_unreachable"version = "1.0.4"@@ -1926,6 +2167,15 @@ dependencies = ["minimal-lexical",]+[[package]]+name = "num-complex"+version = "0.4.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7ae39348c8bc5fbd7f40c727a9925f03517afd2ab27d46702108b6a7e5414c19"+dependencies = [+ "num-traits",+]+[[package]]name = "num-integer"version = "0.1.45"@@ -1975,6 +2225,12 @@ dependencies = ["libc",]+[[package]]+name = "number_prefix"+version = "0.3.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a"+[[package]]name = "number_prefix"version = "0.4.0"@@ -1996,6 +2252,51 @@ dependencies = ["loom",]+[[package]]+name = "onig"+version = "6.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f"+dependencies = [+ "bitflags",+ "libc",+ "once_cell",+ "onig_sys",+]++[[package]]+name = "onig_sys"+version = "69.8.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7b829e3d7e9cc74c7e315ee8edb185bf4190da5acde74afd7fc59c35b1f086e7"+dependencies = [+ "cc",+ "pkg-config",+]++[[package]]+name = "onnxruntime"+version = "0.0.14"+source = "git+https://github.com/Cuely/onnxruntime-rs#1bf6dedaddf7611a61d79e97105f0571251559fe"+dependencies = [+ "lazy_static",+ "ndarray",+ "onnxruntime-sys",+ "thiserror",+ "tracing",+]++[[package]]+name = "onnxruntime-sys"+version = "0.0.14"+source = "git+https://github.com/Cuely/onnxruntime-rs#1bf6dedaddf7611a61d79e97105f0571251559fe"+dependencies = [+ "flate2",+ "tar",+ "ureq",+ "zip",+]+[[package]]name = "oorandom"version = "11.1.3"@@ -2057,7 +2358,7 @@ dependencies = [name = "optics"version = "0.1.0"dependencies = [- "itertools",+ "itertools 0.10.3","lalrpop","lalrpop-util","logos",@@ -2100,7 +2401,7 @@ dependencies = ["libc","redox_syscall","smallvec",- "windows-sys",+ "windows-sys 0.36.1",][[package]]@@ -2109,6 +2410,12 @@ version = "0.1.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "dd59a245c58efa02bd73c9462fd9d4c28952d650092c6bcba43b60b4707171dd"+[[package]]+name = "paste"+version = "1.0.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba"+[[package]]name = "peeking_take_while"version = "0.1.2"@@ -2444,6 +2751,12 @@ dependencies = ["rand_core 0.5.1",]+[[package]]+name = "rawpointer"+version = "0.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"+[[package]]name = "rayon"version = "1.5.3"@@ -2456,6 +2769,17 @@ dependencies = ["rayon-core",]+[[package]]+name = "rayon-cond"+version = "0.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fd1259362c9065e5ea39a789ef40b1e3fd934c94beb7b5ab3ac6629d3b5e7cb7"+dependencies = [+ "either",+ "itertools 0.8.2",+ "rayon",+]+[[package]]name = "rayon-core"version = "1.9.3"@@ -2650,7 +2974,7 @@ dependencies = ["base64","bytes","chrono",- "digest",+ "digest 0.9.0","futures","hex","hmac",@@ -2663,7 +2987,7 @@ dependencies = ["rusoto_credential","rustc_version","serde",- "sha2",+ "sha2 0.9.9","tokio",]@@ -2753,7 +3077,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2"dependencies = ["lazy_static",- "windows-sys",+ "windows-sys 0.36.1",][[package]]@@ -2902,13 +3226,24 @@ version = "0.9.9"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800"dependencies = [- "block-buffer",+ "block-buffer 0.9.0","cfg-if","cpufeatures",- "digest",+ "digest 0.9.0","opaque-debug",]+[[package]]+name = "sha2"+version = "0.10.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0"+dependencies = [+ "cfg-if",+ "cpufeatures",+ "digest 0.10.6",+]+[[package]]name = "sharded-slab"version = "0.1.4"@@ -2976,6 +3311,18 @@ dependencies = ["lock_api",]+[[package]]+name = "spm_precompiled"+version = "0.1.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326"+dependencies = [+ "base64",+ "nom",+ "serde",+ "unicode-segmentation",+]+[[package]]name = "stable_deref_trait"version = "1.2.0"@@ -3008,6 +3355,18 @@ dependencies = ["quote",]+[[package]]+name = "strsim"+version = "0.8.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"++[[package]]+name = "strsim"+version = "0.9.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6446ced80d6c486436db5c078dde11a9f73d42b57fb273121e160b84f63d894c"+[[package]]name = "strsim"version = "0.10.0"@@ -3057,7 +3416,7 @@ dependencies = ["fastfield_codecs","fs2","htmlescape",- "itertools",+ "itertools 0.10.3","levenshtein_automata","log","lru",@@ -3129,6 +3488,17 @@ version = "1.0.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"+[[package]]+name = "tar"+version = "0.4.38"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4b55807c0344e1e6c04d7c965f5289c39a8d94ae23ed5c0b57aabac549f871c6"+dependencies = [+ "filetime",+ "libc",+ "xattr",+]+[[package]]name = "tempfile"version = "3.3.0"@@ -3311,6 +3681,41 @@ version = "0.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"+[[package]]+name = "tokenizers"+version = "0.13.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f4ff2dd291eac98dcea13e8cf7a0b28c373a90dc9210ccdab0fa9e69ee0cac69"+dependencies = [+ "aho-corasick",+ "cached-path",+ "clap 2.34.0",+ "derive_builder",+ "dirs",+ "esaxx-rs",+ "getrandom 0.2.6",+ "indicatif 0.15.0",+ "itertools 0.9.0",+ "lazy_static",+ "log",+ "macro_rules_attribute",+ "onig",+ "paste",+ "rand 0.8.5",+ "rayon",+ "rayon-cond",+ "regex",+ "regex-syntax",+ "reqwest",+ "serde",+ "serde_json",+ "spm_precompiled",+ "thiserror",+ "unicode-normalization-alignments",+ "unicode-segmentation",+ "unicode_categories",+]+[[package]]name = "tokio"version = "1.19.2"@@ -3559,6 +3964,21 @@ dependencies = ["tinyvec",]+[[package]]+name = "unicode-normalization-alignments"+version = "0.1.12"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de"+dependencies = [+ "smallvec",+]++[[package]]+name = "unicode-segmentation"+version = "1.10.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0fdbf052a0783de01e944a6ce7a8cb939e295b1e7be835a1112c3b9a7f047a5a"+[[package]]name = "unicode-width"version = "0.1.9"@@ -3571,12 +3991,35 @@ version = "0.2.4"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c"+[[package]]+name = "unicode_categories"+version = "0.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"+[[package]]name = "untrusted"version = "0.7.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"+[[package]]+name = "ureq"+version = "2.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b97acb4c28a254fd7a4aeec976c46a7fa404eac4d7c134b30c75144846d7cb8f"+dependencies = [+ "base64",+ "chunked_transfer",+ "flate2",+ "log",+ "once_cell",+ "rustls",+ "url",+ "webpki",+ "webpki-roots",+]+[[package]]name = "url"version = "2.2.2"@@ -3635,6 +4078,12 @@ version = "0.2.15"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"+[[package]]+name = "vec_map"+version = "0.8.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"+[[package]]name = "version_check"version = "0.9.4"@@ -3766,6 +4215,15 @@ dependencies = ["untrusted",]+[[package]]+name = "webpki-roots"+version = "0.22.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87"+dependencies = [+ "webpki",+]+[[package]]name = "weezl"version = "0.1.7"@@ -3819,43 +4277,100 @@ version = "0.36.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2"dependencies = [- "windows_aarch64_msvc",- "windows_i686_gnu",- "windows_i686_msvc",- "windows_x86_64_gnu",- "windows_x86_64_msvc",+ "windows_aarch64_msvc 0.36.1",+ "windows_i686_gnu 0.36.1",+ "windows_i686_msvc 0.36.1",+ "windows_x86_64_gnu 0.36.1",+ "windows_x86_64_msvc 0.36.1",]+[[package]]+name = "windows-sys"+version = "0.42.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7"+dependencies = [+ "windows_aarch64_gnullvm",+ "windows_aarch64_msvc 0.42.0",+ "windows_i686_gnu 0.42.0",+ "windows_i686_msvc 0.42.0",+ "windows_x86_64_gnu 0.42.0",+ "windows_x86_64_gnullvm",+ "windows_x86_64_msvc 0.42.0",+]++[[package]]+name = "windows_aarch64_gnullvm"+version = "0.42.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e"+[[package]]name = "windows_aarch64_msvc"version = "0.36.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47"+[[package]]+name = "windows_aarch64_msvc"+version = "0.42.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4"+[[package]]name = "windows_i686_gnu"version = "0.36.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6"+[[package]]+name = "windows_i686_gnu"+version = "0.42.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7"+[[package]]name = "windows_i686_msvc"version = "0.36.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024"+[[package]]+name = "windows_i686_msvc"+version = "0.42.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246"+[[package]]name = "windows_x86_64_gnu"version = "0.36.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1"+[[package]]+name = "windows_x86_64_gnu"+version = "0.42.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed"++[[package]]+name = "windows_x86_64_gnullvm"+version = "0.42.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028"+[[package]]name = "windows_x86_64_msvc"version = "0.36.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680"+[[package]]+name = "windows_x86_64_msvc"+version = "0.42.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5"+[[package]]name = "winreg"version = "0.10.1"@@ -3874,6 +4389,15 @@ dependencies = ["tap",]+[[package]]+name = "xattr"+version = "0.2.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6d1526bbe5aaeb5eb06885f4d987bcdfa5e23187055de9b83fe00156a821fabc"+dependencies = [+ "libc",+]+[[package]]name = "xml-rs"version = "0.8.4"@@ -3886,6 +4410,29 @@ version = "1.5.7"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "c394b5bd0c6f669e7275d9c20aa90ae064cb22e75a1cad54e1b34088034b149f"+[[package]]+name = "zip"+version = "0.5.13"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "93ab48844d61251bb3835145c521d88aa4031d7139e8485990f60ca911fa0815"+dependencies = [+ "byteorder",+ "bzip2",+ "crc32fast",+ "flate2",+ "thiserror",+ "time 0.1.44",+]++[[package]]+name = "zip-extensions"+version = "0.6.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a64c3c977bc3434ce2d4bcea8ad3c644672de0f2c402b72b9171ca80a8885d14"+dependencies = [+ "zip",+]+[[package]]name = "zstd-sys"version = "2.0.1+zstd.1.5.2"diff --git a/Cargo.toml b/Cargo.tomlindex 684df09b..e32ec6d0 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -64,4 +64,6 @@ criterion = "0.3.6"maplit = "1.0.2"rand = "0.8.5"crossbeam-channel = "0.5.6"-num_cpus = "1.15.0"\ No newline at end of file+num_cpus = "1.15.0"+onnxruntime = { git = "https://github.com/Cuely/onnxruntime-rs" }+tokenizers = "0.13.2"\ No newline at end of filediff --git a/core/Cargo.toml b/core/Cargo.tomlindex 6586779f..34382625 100644--- a/core/Cargo.toml+++ b/core/Cargo.toml@@ -69,6 +69,8 @@ once_cell = { workspace = true }optics = { path = "../optics" }crossbeam-channel = { workspace = true }num_cpus = { workspace = true }+onnxruntime = { workspace = true }+tokenizers = { workspace = true }[dev-dependencies]criterion = { workspace = true }diff --git a/core/examples/print_approx_harmonic_centrality.rs b/core/examples/print_approx_harmonic_centrality.rsindex b11d723e..28119ebc 100644--- a/core/examples/print_approx_harmonic_centrality.rs+++ b/core/examples/print_approx_harmonic_centrality.rs@@ -96,10 +96,7 @@ fn print_top_nodes(let scorer = store.online_harmonic.scorer(&liked_nodes, &[]);let top_nodes = get_top_nodes(scorer, top_n, nodes);- println!(- "top {} sites for these liked sites {:?}",- top_n, liked_sites- );+ println!("top {top_n} sites for these liked sites {liked_sites:?}");print_nodes(&top_nodes, id2node);println!();diff --git a/core/examples/print_inbound_links.rs b/core/examples/print_inbound_links.rsindex 0b11f933..e44dcbba 100644--- a/core/examples/print_inbound_links.rs+++ b/core/examples/print_inbound_links.rs@@ -25,7 +25,7 @@ pub fn main() {"onlinester.ga","GroomersNew.ga",] {- println!("{}:", host);+ println!("{host}:");for edge in graph.ingoing_edges(Node::from(host)) {println!("{} -> {} ({})", edge.from.name, edge.to.name, edge.label);}diff --git a/core/src/entrypoint/frontend.rs b/core/src/entrypoint/frontend.rsindex a7137f6e..64f633dd 100644--- a/core/src/entrypoint/frontend.rs+++ b/core/src/entrypoint/frontend.rs@@ -18,8 +18,13 @@ use anyhow::Result;use crate::frontend::router;-pub async fn run(queries_csv_path: &str, host: &str, shards: Vec<Vec<String>>) -> Result<()> {- let app = router(queries_csv_path, shards)?;+pub async fn run(+ queries_csv_path: &str,+ crossencoder_model_path: &str,+ host: &str,+ shards: Vec<Vec<String>>,+) -> Result<()> {+ let app = router(queries_csv_path, crossencoder_model_path, shards)?;let addr = host.parse()?;tracing::info!("listening on {}", addr);axum::Server::bind(&addr)diff --git a/core/src/frontend/mod.rs b/core/src/frontend/mod.rsindex bcb58874..66f9891d 100644--- a/core/src/frontend/mod.rs+++ b/core/src/frontend/mod.rs@@ -19,6 +19,7 @@ use tower_http::{compression::CompressionLayer, services::ServeDir};use crate::{autosuggest::Autosuggest,+ ranking::models::cross_encoder::CrossEncoderModel,searcher::{DistributedSearcher, Shard},};use anyhow::Result;@@ -57,7 +58,7 @@ whereOk(html) => Html(html).into_response(),Err(err) => (StatusCode::INTERNAL_SERVER_ERROR,- format!("Failed to render template. Error: {}", err),+ format!("Failed to render template. Error: {err}"),).into_response(),}@@ -74,7 +75,11 @@ pub async fn favicon() -> impl IntoResponse {.unwrap()}-pub fn router(queries_csv_path: &str, shards: Vec<Vec<String>>) -> Result<Router> {+pub fn router(+ queries_csv_path: &str,+ crossencoder_model_path: &str,+ shards: Vec<Vec<String>>,+) -> Result<Router> {let shards: Vec<_> = shards.into_iter().enumerate()@@ -82,7 +87,8 @@ pub fn router(queries_csv_path: &str, shards: Vec<Vec<String>>) -> Result<Router.collect();let autosuggest = Autosuggest::load_csv(queries_csv_path)?;- let searcher = DistributedSearcher::new(shards);+ let crossencoder = CrossEncoderModel::open(crossencoder_model_path)?;+ let searcher = DistributedSearcher::new(shards, crossencoder);let state = Arc::new(State {searcher,@@ -106,7 +112,7 @@ pub fn router(queries_csv_path: &str, shards: Vec<Vec<String>>) -> Result<Router|error: std::io::Error| async move {(StatusCode::INTERNAL_SERVER_ERROR,- format!("Unhandled internal error: {}", error),+ format!("Unhandled internal error: {error}"),)},))diff --git a/core/src/hyperloglog.rs b/core/src/hyperloglog.rsindex 7f1f0b61..9ecc8f75 100644--- a/core/src/hyperloglog.rs+++ b/core/src/hyperloglog.rs@@ -4038,13 +4038,13 @@ use std::array;#[derive(Clone)]pub struct HyperLogLog<const N: usize> {- registers: [u8; N],+ registers: Box<[u8; N]>,}impl<const N: usize> Default for HyperLogLog<N> {fn default() -> Self {Self {- registers: array::from_fn(|_| 0),+ registers: Box::new(array::from_fn(|_| 0)),}}}@@ -4202,7 +4202,7 @@ impl<const N: usize> HyperLogLog<N> {}pub fn registers(&self) -> &[u8] {- &self.registers+ self.registers.as_ref()}}diff --git a/core/src/image_store.rs b/core/src/image_store.rsindex d817417b..c3bb50b7 100644--- a/core/src/image_store.rs+++ b/core/src/image_store.rs@@ -20,7 +20,7 @@ use image::imageops::FilterType;use image::{DynamicImage, ImageOutputFormat};use serde::{de, ser::SerializeStruct, Serialize};use std::collections::HashSet;-use std::io::{Cursor, Read, Seek, SeekFrom};+use std::io::{Cursor, Read, Seek};use std::path::Path;use uuid::Uuid;@@ -295,7 +295,7 @@ impl Image {self.0.write_to(&mut cursor, ImageOutputFormat::Png).unwrap();- cursor.seek(SeekFrom::Start(0)).unwrap();+ cursor.rewind().unwrap();let mut bytes = Vec::new();cursor.read_to_end(&mut bytes).unwrap();diff --git a/core/src/kv/rocksdb_store.rs b/core/src/kv/rocksdb_store.rsindex d2efae6f..655feed2 100644--- a/core/src/kv/rocksdb_store.rs+++ b/core/src/kv/rocksdb_store.rs@@ -82,7 +82,7 @@ whereif let Err(err) = self.flush() {match err.kind() {rocksdb::ErrorKind::NotSupported => {}- _ => panic!("failed to flush: {:?}", err),+ _ => panic!("failed to flush: {err:?}"),}}}diff --git a/core/src/lib.rs b/core/src/lib.rsindex 389190c2..bf9f48a4 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -151,6 +151,7 @@ pub struct HttpConfig {pub struct FrontendConfig {pub queries_csv_path: String,pub host: String,+ pub crossencoder_model_path: String,pub search_servers: Vec<Vec<String>>,}@@ -242,6 +243,12 @@ pub enum Error {#[error("Internal error")]InternalError(String),++ #[error("Huggingface tokenizer")]+ HuggingfaceTokenizer(#[from] tokenizers::Error),++ #[error("Onnx")]+ Onnx(#[from] onnxruntime::OrtError),}pub(crate) type Result<T> = std::result::Result<T, Error>;diff --git a/core/src/main.rs b/core/src/main.rsindex f09e84c9..62fe25cb 100644--- a/core/src/main.rs+++ b/core/src/main.rs@@ -190,6 +190,7 @@ fn main() -> Result<()> {.build()?.block_on(frontend::run(&config.queries_csv_path,+ &config.crossencoder_model_path,&config.host,config.search_servers,))?diff --git a/core/src/mapreduce/manager.rs b/core/src/mapreduce/manager.rsindex ea322173..d2f81d67 100644--- a/core/src/mapreduce/manager.rs+++ b/core/src/mapreduce/manager.rs@@ -115,7 +115,7 @@ impl WorkerPool {.iter().flat_map(|addr| {addr.to_socket_addrs().unwrap_or_else(|_| {- panic!("failed to transform {:?} into a socket address", addr)+ panic!("failed to transform {addr:?} into a socket address")})}).map(|addr| Arc::new(RemoteWorker { addr }))diff --git a/core/src/query/pattern_query.rs b/core/src/query/pattern_query.rsindex 52d2fc45..2fca2ddb 100644--- a/core/src/query/pattern_query.rs+++ b/core/src/query/pattern_query.rs@@ -222,15 +222,13 @@ impl tantivy::query::Weight for PatternWeight {let scorer_opt = self.pattern_scorer(reader, 1.0)?;if scorer_opt.is_none() {return Err(TantivyError::InvalidArgument(format!(- "Document #({}) does not match",- doc+ "Document #({doc}) does not match")));}let mut scorer = scorer_opt.unwrap();if scorer.seek(doc) != doc {return Err(TantivyError::InvalidArgument(format!(- "Document #({}) does not match",- doc+ "Document #({doc}) does not match")));}let fieldnorm_reader = self.fieldnorm_reader(reader)?;diff --git a/core/src/ranking/bm25.rs b/core/src/ranking/bm25.rsindex 7c45a5b9..e79de857 100644--- a/core/src/ranking/bm25.rs+++ b/core/src/ranking/bm25.rs@@ -10,7 +10,7 @@ const K1: Score = 1.2;const B: Score = 0.75;pub(crate) fn idf(doc_freq: u64, doc_count: u64) -> Score {- assert!(doc_count >= doc_freq, "{} >= {}", doc_count, doc_freq);+ assert!(doc_count >= doc_freq, "{doc_count} >= {doc_freq}");let x = ((doc_count - doc_freq) as Score + 0.5) / (doc_freq as Score + 0.5);(1.0 + x).ln()}diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rsindex a6d3bea2..a52b2aa1 100644--- a/core/src/ranking/mod.rs+++ b/core/src/ranking/mod.rs@@ -17,6 +17,7 @@pub mod bm25;pub mod centrality_store;pub mod initial;+pub mod models;pub mod optics;pub mod pipeline;pub mod signal;diff --git a/core/src/ranking/models/cross_encoder.rs b/core/src/ranking/models/cross_encoder.rsnew file mode 100644index 00000000..9ea8f7a2--- /dev/null+++ b/core/src/ranking/models/cross_encoder.rs@@ -0,0 +1,123 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::{path::Path, sync::Mutex};++use itertools::Itertools;+use onnxruntime::{ndarray::ArrayBase, tensor::OrtOwnedTensor, GraphOptimizationLevel};++use crate::Result;++use super::ONNX_ENVIRONMENT;++pub struct CrossEncoderModel {+ tokenizer: tokenizers::Tokenizer,+ session: Mutex<onnxruntime::session::Session<'static>>,+}++impl CrossEncoderModel {+ pub fn open<P: AsRef<Path>>(folder: P) -> Result<Self> {+ let tokenizer = tokenizers::Tokenizer::from_file(folder.as_ref().join("tokenizer.json"))?;++ let session = Mutex::new(+ ONNX_ENVIRONMENT+ .new_session_builder()?+ .with_optimization_level(GraphOptimizationLevel::All)?+ .with_number_threads(1)?+ .with_model_from_file(folder.as_ref().join("model.onnx"))?,+ );++ Ok(Self { tokenizer, session })+ }+}++pub trait CrossEncoder: Send + Sync {+ fn run(&self, query: &str, body: &str) -> f64;+}++pub struct DummyCrossEncoder {}++impl CrossEncoder for DummyCrossEncoder {+ fn run(&self, _query: &str, _body: &str) -> f64 {+ 1.0+ }+}++impl CrossEncoder for CrossEncoderModel {+ fn run(&self, query: &str, body: &str) -> f64 {+ let encoded = self.tokenizer.encode((query, body), true).unwrap();++ let ids = encoded.get_ids().iter().map(|i| *i as i64).collect_vec();++ let attention_mask = encoded+ .get_attention_mask()+ .iter()+ .map(|i| *i as i64)+ .collect_vec();++ let type_ids = encoded+ .get_type_ids()+ .iter()+ .map(|i| *i as i64)+ .collect_vec();++ let num_tokens = ids.len();++ let mut sess = self.session.lock().unwrap();++ let res: Vec<OrtOwnedTensor<f64, _>> = sess+ .run(vec![+ ArrayBase::from_vec(ids)+ .into_shape((1, num_tokens))+ .unwrap(),+ ArrayBase::from_vec(attention_mask)+ .into_shape((1, num_tokens))+ .unwrap(),+ ArrayBase::from_vec(type_ids)+ .into_shape((1, num_tokens))+ .unwrap(),+ ])+ .unwrap();++ res[0][[0, 0]]+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn sanity_check() {+ match CrossEncoderModel::open("../data/cross_encoder") {+ Ok(model) => {+ assert!(+ model.run(+ "how many people live in paris",+ "there are currently 1234 people living in paris"+ ) > model.run(+ "how many people live in paris",+ "I really like cake and my favorite cake is probably brownie"+ )+ );+ }+ Err(err) => {+ dbg!(err);+ // panic!();+ }+ }+ }+}diff --git a/core/src/ranking/models/mod.rs b/core/src/ranking/models/mod.rsnew file mode 100644index 00000000..13e94d77--- /dev/null+++ b/core/src/ranking/models/mod.rs@@ -0,0 +1,28 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use onnxruntime::{environment::Environment, LoggingLevel};++pub mod cross_encoder;++pub static ONNX_ENVIRONMENT: once_cell::sync::Lazy<Environment> =+ once_cell::sync::Lazy::new(|| {+ Environment::builder()+ .with_name("ranking")+ .with_log_level(LoggingLevel::Info)+ .build()+ .unwrap()+ });diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rsindex f35e9751..2d081fdb 100644--- a/core/src/ranking/pipeline.rs+++ b/core/src/ranking/pipeline.rs@@ -14,17 +14,19 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::cmp::Ordering;+use std::{cmp::Ordering, sync::Arc};+use optics::ast::{RankingCoeff, RankingTarget};use serde::{Deserialize, Serialize};use crate::{inverted_index::WebsitePointer,schema::{FastField, Field, TextField, ALL_FIELDS, FLOAT_SCALING},searcher::SearchQuery,+ Result,};-use super::SignalAggregator;+use super::{models::cross_encoder::CrossEncoder, SignalAggregator};pub trait AsRankingWebsite: Clone {fn as_ranking(&self) -> &RankingWebsite;@@ -105,26 +107,51 @@ impl RankingWebsite {trait Scorer<T: AsRankingWebsite>: Send + Sync {fn score(&self, websites: &mut [T]);+ fn set_coefficients(&mut self, coefficients: &[RankingCoeff]);+ fn set_query_info(&mut self, _query: &SearchQuery) {}}-struct ReRanker {- crossencoder: f64,+struct ReRanker<M: CrossEncoder> {+ coefficient: f64,+ model: Arc<M>,prev_score: f64,+ query: String,}-impl Default for ReRanker {- fn default() -> Self {+impl<M: CrossEncoder> ReRanker<M> {+ fn new(model: Arc<M>) -> Self {Self {- crossencoder: 1.0,+ coefficient: 1.0,+ model,prev_score: 1.0,+ query: String::new(),}}}-impl<T: AsRankingWebsite> Scorer<T> for ReRanker {+impl<T: AsRankingWebsite, M: CrossEncoder> Scorer<T> for ReRanker<M> {fn score(&self, websites: &mut [T]) {- // TODO: Implement actual scoring- // todo!();+ for website in websites {+ let mut website = website.as_mut_ranking();+ website.score = self.prev_score * website.score+ + self.coefficient * self.model.run(&self.query, &website.clean_body);+ }+ }++ fn set_query_info(&mut self, query: &SearchQuery) {+ self.query = query.query.clone();+ }++ fn set_coefficients(&mut self, coefficients: &[RankingCoeff]) {+ for coeff in coefficients {+ if let RankingTarget::Signal(signal) = &coeff.target {+ match signal.as_str() {+ "prev_score" => self.prev_score = coeff.score,+ "crossencoder" => self.coefficient = coeff.score,+ _ => {}+ }+ }+ }}}@@ -150,6 +177,18 @@ impl<T: AsRankingWebsite> Scorer<T> for PrioritizeText {website.as_mut_ranking().score = self.bm25 * bm25 + self.prev_score * prev_score;}}++ fn set_coefficients(&mut self, coefficients: &[RankingCoeff]) {+ for coeff in coefficients {+ if let RankingTarget::Signal(signal) = &coeff.target {+ match signal.as_str() {+ "prev_score" => self.prev_score = coeff.score,+ "bm25" => self.bm25 = coeff.score,+ _ => {}+ }+ }+ }+ }}enum Prev<T: AsRankingWebsite> {@@ -222,6 +261,14 @@ impl<T: AsRankingWebsite> RankingStage<T> {.take(top_n).collect()}++ fn set_query_info(&mut self, query: &SearchQuery) {+ self.scorer.set_query_info(query);+ match &mut self.prev {+ Prev::Initial => {}+ Prev::Node(prev) => prev.set_query_info(query),+ }+ }}pub struct RankingPipeline<T: AsRankingWebsite> {@@ -231,9 +278,9 @@ pub struct RankingPipeline<T: AsRankingWebsite> {}impl<T: AsRankingWebsite> RankingPipeline<T> {- fn create() -> Self {+ fn create<M: CrossEncoder + 'static>(crossencoder: Arc<M>) -> Result<Self> {let last_stage = RankingStage {- scorer: Box::<ReRanker>::default(),+ scorer: Box::new(ReRanker::new(crossencoder)),prev: Prev::Node(Box::new(RankingStage {scorer: Box::<PrioritizeText>::default(),prev: Prev::Initial,@@ -244,17 +291,43 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {top_n: 50,};- Self {+ Ok(Self {last_stage,offset: 0,top_n: 0,- }+ })}- pub fn for_query(query: &mut SearchQuery) -> Self {- dbg!(&query.optic_program);+ pub fn for_query<M: CrossEncoder + 'static>(+ query: &mut SearchQuery,+ crossencoder: Arc<M>,+ ) -> Result<Self> {+ let mut pipeline = Self::create(crossencoder)?;+ if let Some(optic) = &query.optic_program {+ let optic = optics::parse(optic)?;+ if let Some(optic_pipeline) = optic.pipeline {+ let mut stage = Some(&mut pipeline.last_stage);++ // skip(1) since first stage is collector+ for optic_stage in optic_pipeline.stages.into_iter().skip(1).rev() {+ match stage {+ Some(inner_stage) => {+ inner_stage+ .scorer+ .set_coefficients(&optic_stage.coefficients);++ match &mut inner_stage.prev {+ Prev::Initial => stage = None,+ Prev::Node(next) => stage = Some(next),+ }+ }+ None => break,+ }+ }+ }+ }- let mut pipeline = Self::create();+ pipeline.last_stage.set_query_info(query);pipeline.offset = query.offset;pipeline.top_n = query.num_results;@@ -262,7 +335,7 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {query.num_results = pipeline.collector_top_n();query.offset = pipeline.collector_offset();- pipeline+ Ok(pipeline)}pub fn apply(mut self, websites: Vec<T>) -> Vec<T> {@@ -288,6 +361,7 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {mod tests {use itertools::Itertools;+ use crate::ranking::models::cross_encoder::DummyCrossEncoder;use crate::{collector::Hashes, inverted_index::DocAddress, prehashed::Prehashed,ranking::initial::Score,@@ -330,9 +404,13 @@ mod tests {#[test]fn simple() {- let pipeline = RankingPipeline::for_query(&mut SearchQuery {- ..Default::default()- });+ let pipeline = RankingPipeline::for_query(+ &mut SearchQuery {+ ..Default::default()+ },+ Arc::new(DummyCrossEncoder {}),+ )+ .unwrap();assert_eq!(pipeline.collector_top_n(), 200);let sample = sample_websites(pipeline.collector_top_n());@@ -353,10 +431,14 @@ mod tests {#[test]fn offsets() {- let pipeline = RankingPipeline::for_query(&mut SearchQuery {- offset: 0,- ..Default::default()- });+ let pipeline = RankingPipeline::for_query(+ &mut SearchQuery {+ offset: 0,+ ..Default::default()+ },+ Arc::new(DummyCrossEncoder {}),+ )+ .unwrap();let sample: Vec<_> =sample_websites(pipeline.collector_top_n() + pipeline.collector_offset())@@ -366,10 +448,14 @@ mod tests {let mut prev: Vec<_> = pipeline.apply(sample);for offset in 1..1_000 {- let pipeline = RankingPipeline::for_query(&mut SearchQuery {- offset,- ..Default::default()- });+ let pipeline = RankingPipeline::for_query(+ &mut SearchQuery {+ offset,+ ..Default::default()+ },+ Arc::new(DummyCrossEncoder {}),+ )+ .unwrap();let sample: Vec<_> =sample_websites(pipeline.collector_top_n() + pipeline.collector_offset())@@ -386,24 +472,18 @@ mod tests {.any(|r| r.pointer.address.doc_id == first.pointer.address.doc_id));}- // assert_eq!(- // prev.iter()- // .map(|p| usize::from(- // res.iter()- // .any(|r| r.pointer.address.doc_id == p.pointer.address.doc_id)- // ))- // .sum::<usize>(),- // 19,- // "Only the top result from previous offset should be removed in current ranking"- // );-prev = res;}- let pipeline = RankingPipeline::for_query(&mut SearchQuery {- offset: 0,- ..Default::default()- });+ let pipeline = RankingPipeline::for_query(+ &mut SearchQuery {+ offset: 0,+ ..Default::default()+ },+ Arc::new(DummyCrossEncoder {}),+ )+ .unwrap();+let sample: Vec<_> =sample_websites(pipeline.collector_top_n() + pipeline.collector_offset()).into_iter()@@ -411,10 +491,14 @@ mod tests {.collect();let mut prev: Vec<_> = pipeline.apply(sample);for p in 1..100 {- let pipeline = RankingPipeline::for_query(&mut SearchQuery {- offset: p * 20,- ..Default::default()- });+ let pipeline = RankingPipeline::for_query(+ &mut SearchQuery {+ offset: p * 20,+ ..Default::default()+ },+ Arc::new(DummyCrossEncoder {}),+ )+ .unwrap();let sample: Vec<_> =sample_websites(pipeline.collector_top_n() + pipeline.collector_offset())@@ -436,29 +520,29 @@ mod tests {#[test]fn multistage_coefficients() {- let pipeline = RankingPipeline::for_query(&mut SearchQuery {- optic_program: Some(- r#"+ let pipeline = RankingPipeline::for_query(+ &mut SearchQuery {+ optic_program: Some(+ r#"RankingPipeline {Stage {},Stage {- Ranking {- Signal("bm25", 3),- Signal("prev_score", 2),- },+ Ranking{Signal("bm25"), 3},+ Ranking{Signal("prev_score"), 2},},Stage {- Ranking {- Signal("crossencoder", 4),- Signal("prev_score", 3),- },+ Ranking{Signal("crossencoder"), 4},+ Ranking{Signal("prev_score"), 3},},}"#- .to_string(),- ),- ..Default::default()- });+ .to_string(),+ ),+ ..Default::default()+ },+ Arc::new(DummyCrossEncoder {}),+ )+ .unwrap();let w = RankingWebsite {pointer: WebsitePointer {@@ -487,16 +571,43 @@ mod tests {score: 1.0,};+ let res = pipeline.apply(vec![w.clone()]);+ assert_eq!(res[0].score, 19.0);++ let pipeline = RankingPipeline::for_query(+ &mut SearchQuery {+ optic_program: Some(+ r#"+ RankingPipeline {+ Stage {},+ Stage {+ Ranking{Signal("bm25"), 3},+ Ranking{Signal("prev_score"), 2},+ },+ Stage {+ Ranking{Signal("crossencoder"), 4},+ Ranking{Signal("prev_score"), 3},+ },+ }+ "#+ .to_string(),+ ),+ ..Default::default()+ },+ Arc::new(DummyCrossEncoder {}),+ )+ .unwrap();+let mut test = [w.clone()];pipeline.last_stage.scorer.score(&mut test);- assert_eq!(test[0].score, 5.0);+ assert_eq!(test[0].score, 7.0);let mut test = [w];if let Prev::Node(prev) = pipeline.last_stage.prev {prev.scorer.score(&mut test);- assert_eq!(test[0].score, 3.0);+ assert_eq!(test[0].score, 5.0);}}}diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rsindex 3c80f5cf..786d545b 100644--- a/core/src/searcher/distributed.rs+++ b/core/src/searcher/distributed.rs@@ -18,12 +18,15 @@ use crate::{collector::{self, BucketCollector},exponential_backoff::ExponentialBackoff,inverted_index::{self},- ranking::pipeline::{AsRankingWebsite, RankingPipeline, RankingWebsite},+ ranking::{+ models::cross_encoder::CrossEncoderModel,+ pipeline::{AsRankingWebsite, RankingPipeline, RankingWebsite},+ },search_prettifier::DisplayedWebpage,searcher::PrettifiedWebsitesResult,};-use std::{net::SocketAddr, time::Instant};+use std::{net::SocketAddr, sync::Arc, time::Instant};use futures::stream::FuturesUnordered;use futures::StreamExt;@@ -47,6 +50,9 @@ pub enum Error {#[error("Query cannot be empty")]EmptyQuery,++ #[error("Internal error")]+ InternalError(#[from] crate::Error),}impl RemoteSearcher {@@ -171,6 +177,7 @@ pub enum Request {pub struct DistributedSearcher {shards: Vec<Shard>,+ cross_encoder: Arc<CrossEncoderModel>,}#[derive(Clone)]@@ -204,8 +211,11 @@ impl collector::Doc for ScoredWebsitePointer {}impl DistributedSearcher {- pub fn new(shards: Vec<Shard>) -> Self {- Self { shards }+ pub fn new(shards: Vec<Shard>, model: CrossEncoderModel) -> Self {+ Self {+ shards,+ cross_encoder: Arc::new(model),+ }}pub async fn search(&self, query: &SearchQuery) -> Result<PrettifiedSearchResult> {@@ -217,7 +227,7 @@ impl DistributedSearcher {let mut search_query = query.clone();let pipeline: RankingPipeline<ScoredWebsitePointer> =- RankingPipeline::for_query(&mut search_query);+ RankingPipeline::for_query(&mut search_query, self.cross_encoder.clone())?;// search shardslet initial_results = selfdiff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex deca8413..05dfb3f0 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -15,7 +15,6 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use std::str::FromStr;-use std::time::Instant;use optics::Optic;use serde::{Deserialize, Serialize};@@ -28,7 +27,7 @@ use crate::index::Index;use crate::query::Query;use crate::ranking::centrality_store::CentralityStore;use crate::ranking::optics::CreateAggregator;-use crate::ranking::pipeline::{RankingPipeline, RankingWebsite};+use crate::ranking::pipeline::RankingWebsite;use crate::ranking::{online_centrality_scorer, Ranker, SignalAggregator};use crate::spell::Correction;use crate::webgraph::centrality::topic::TopicCentrality;@@ -36,7 +35,10 @@ use crate::webpage::region::Region;use crate::webpage::Url;use crate::{inverted_index, Error, Result};-use super::{InitialSearchResult, SearchQuery, SearchResult, Sidebar, WebsitesResult};+use super::{InitialSearchResult, SearchQuery, SearchResult, Sidebar};++#[cfg(test)]+use super::WebsitesResult;const STACKOVERFLOW_SIDEBAR_SCORE_THRESHOLD: f64 = 250.0;@@ -309,12 +311,24 @@ impl LocalSearcher {self.index.retrieve_websites(websites, &query)}+ #[cfg(test)]pub fn search(&self, query: &SearchQuery) -> Result<SearchResult> {- let start = Instant::now();+ use std::{sync::Arc, time::Instant};+ use crate::ranking::{+ models::cross_encoder::{CrossEncoderModel, DummyCrossEncoder},+ pipeline::RankingPipeline,+ };++ let start = Instant::now();let mut search_query = query.clone();- let pipeline = RankingPipeline::for_query(&mut search_query);+ let pipeline = match CrossEncoderModel::open("data/cross_encoder") {+ Ok(model) => RankingPipeline::for_query(&mut search_query, Arc::new(model))?,+ Err(_) => {+ RankingPipeline::for_query(&mut search_query, Arc::new(DummyCrossEncoder {}))?+ }+ };let initial_result = self.search_initial(&search_query, true)?;diff --git a/core/src/snippet.rs b/core/src/snippet.rsindex e292cc3c..e4c6bf6c 100644--- a/core/src/snippet.rs+++ b/core/src/snippet.rs@@ -127,9 +127,9 @@ impl Snippet {////// Fragments must be valid in the sense that `&text[fragment.start..fragment.stop]`\/// has to be a valid string.-fn search_fragments<'a>(+fn search_fragments(tokenizer: &TextAnalyzer,- text: &'a str,+ text: &str,terms: &BTreeMap<String, Score>,max_num_chars: usize,) -> Vec<FragmentCandidate> {@@ -373,11 +373,10 @@ Survey in 2016, 2017, and 2018."#;<title>Website for runners</title></head><body>- {}+ {TEST_TEXT}</body></html>- "#,- TEST_TEXT+ "#),"https://www.example.com",))@@ -413,11 +412,10 @@ Survey in 2016, 2017, and 2018."#;<title>Website for runners</title></head><body>- {}+ {TEST_TEXT}</body></html>- "#,- TEST_TEXT+ "#),"https://www.example.com",))@@ -623,11 +621,10 @@ Survey in 2016, 2017, and 2018."#;<title>Website for runners</title></head><body>- {}+ {TEST_TEXT}</body></html>- "#,- TEST_TEXT+ "#),"https://www.example.com",))diff --git a/core/src/warc.rs b/core/src/warc.rsindex a709178c..7bee7682 100644--- a/core/src/warc.rs+++ b/core/src/warc.rs@@ -18,7 +18,7 @@ use crate::exponential_backoff::ExponentialBackoff;use crate::{Error, Result, WarcSource};use std::collections::BTreeMap;use std::fs::File;-use std::io::{BufRead, BufReader, Cursor, Read, Seek, SeekFrom, Write};+use std::io::{BufRead, BufReader, Cursor, Read, Seek, Write};use std::path::Path;use std::time::Duration;@@ -125,7 +125,7 @@ impl WarcFile {let f = File::open(Path::new(folder).join(name))?;let mut reader = BufReader::new(f);- buf.seek(SeekFrom::Start(0))?;+ buf.rewind()?;std::io::copy(&mut reader, buf)?;Ok(())}@@ -151,7 +151,7 @@ impl WarcFile {let mut stream = res.bytes_stream();- buf.seek(SeekFrom::Start(0))?;+ buf.rewind()?;while let Some(chunk) = stream.next().await {std::io::copy(&mut &chunk?[..], buf)?;}diff --git a/core/src/webgraph/centrality/online_harmonic.rs b/core/src/webgraph/centrality/online_harmonic.rsindex 87acd6a2..433443fd 100644--- a/core/src/webgraph/centrality/online_harmonic.rs+++ b/core/src/webgraph/centrality/online_harmonic.rs@@ -30,7 +30,6 @@ use std::path::Path;use std::sync::{Arc, Mutex};use itertools::Itertools;-use rayon::prelude::*;use serde::{Deserialize, Serialize};use crate::intmap::IntMap;diff --git a/core/src/webgraph/executor.rs b/core/src/webgraph/executor.rsindex a5041424..46226517 100644--- a/core/src/webgraph/executor.rs+++ b/core/src/webgraph/executor.rs@@ -38,7 +38,7 @@ impl Executor {pub fn with_threads(num_threads: usize, prefix: &'static str) -> Result<Executor> {let pool = ThreadPoolBuilder::new().num_threads(num_threads)- .thread_name(move |num| format!("{}{}", prefix, num))+ .thread_name(move |num| format!("{prefix}{num}")).build()?;Ok(Executor::ThreadPool(pool))}diff --git a/scripts/export_crossencoder b/scripts/export_crossencoderindex fba0becb..83316637 100755--- a/scripts/export_crossencoder+++ b/scripts/export_crossencoder@@ -1,4 +1,13 @@-#!/bin/bash-pip3 install transformers[onnx] torch-python3 -m transformers.onnx --model=cross-encoder/ms-marco-MiniLM-L-6-v2 data/-mv data/model.onnx data/crossencoder.onnx+#!/usr/bin/env python3+import os+import transformers++MODEL = "cross-encoder/ms-marco-MiniLM-L-6-v2"++os.system("pip3 install transformers[onnx] torch")+os.system("mkdir data/cross_encoder")+os.system(+ f"python3 -m transformers.onnx --feature sequence-classification --model={MODEL} data/cross_encoder")++tokenizer = transformers.AutoTokenizer.from_pretrained(MODEL)+tokenizer.save_pretrained("data/cross_encoder")