Skip to content

web search done right

crossencoder ranking (finally!)

Mikkel Denker0a4650440217
diff --git a/Cargo.lock b/Cargo.lock
index a4f69b8d..28ebba62 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -327,6 +327,15 @@ dependencies = [
"generic-array",
]
+[[package]]
+name = "block-buffer"
+version = "0.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e"
+dependencies = [
+ "generic-array",
+]
+
[[package]]
name = "bstr"
version = "0.2.17"
@@ -384,6 +393,29 @@ dependencies = [
"pkg-config",
]
+[[package]]
+name = "cached-path"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5f1c56d30236522ab3393a08746b138d4e16372001f42d29c88d513aeb8ab7ef"
+dependencies = [
+ "flate2",
+ "fs2",
+ "glob",
+ "indicatif 0.16.2",
+ "log",
+ "rand 0.8.5",
+ "reqwest",
+ "serde",
+ "serde_json",
+ "sha2 0.10.6",
+ "tar",
+ "tempfile",
+ "thiserror",
+ "zip",
+ "zip-extensions",
+]
+
[[package]]
name = "cast"
version = "0.3.0"
@@ -435,6 +467,12 @@ dependencies = [
"winapi",
]
+[[package]]
+name = "chunked_transfer"
+version = "1.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cca491388666e04d7248af3f60f0c40cfb0991c72205595d7c396e3510207d1a"
+
[[package]]
name = "clang-sys"
version = "1.3.3"
@@ -452,9 +490,13 @@ version = "2.34.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
dependencies = [
+ "ansi_term",
+ "atty",
"bitflags",
+ "strsim 0.8.0",
"textwrap 0.11.0",
"unicode-width",
+ "vec_map",
]
[[package]]
@@ -469,7 +511,7 @@ dependencies = [
"clap_lex",
"indexmap",
"lazy_static",
- "strsim",
+ "strsim 0.10.0",
"termcolor",
"textwrap 0.15.0",
]
@@ -520,7 +562,9 @@ dependencies = [
"encode_unicode",
"libc",
"once_cell",
+ "regex",
"terminal_size",
+ "unicode-width",
"winapi",
]
@@ -575,7 +619,7 @@ dependencies = [
"clap 2.34.0",
"criterion-plot",
"csv",
- "itertools",
+ "itertools 0.10.3",
"lazy_static",
"num-traits",
"oorandom",
@@ -597,7 +641,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2673cc8207403546f45f5fd319a974b1e6983ad1a3ee7e6041650013be041876"
dependencies = [
"cast",
- "itertools",
+ "itertools 0.10.3",
]
[[package]]
@@ -651,6 +695,16 @@ version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"
+[[package]]
+name = "crypto-common"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
+dependencies = [
+ "generic-array",
+ "typenum",
+]
+
[[package]]
name = "crypto-mac"
version = "0.11.1"
@@ -736,8 +790,8 @@ dependencies = [
"html-escape",
"http",
"image",
- "indicatif",
- "itertools",
+ "indicatif 0.16.2",
+ "itertools 0.10.3",
"kuchiki",
"logos",
"maplit",
@@ -746,6 +800,7 @@ dependencies = [
"min-max-heap",
"num_cpus",
"once_cell",
+ "onnxruntime",
"optics",
"parse_wiki_text",
"quick-xml",
@@ -762,6 +817,7 @@ dependencies = [
"serde_urlencoded",
"tantivy",
"thiserror",
+ "tokenizers",
"tokio",
"toml",
"tower-http",
@@ -772,6 +828,41 @@ dependencies = [
"whatlang",
]
+[[package]]
+name = "darling"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0d706e75d87e35569db781a9b5e2416cff1236a47ed380831f959382ccd5f858"
+dependencies = [
+ "darling_core",
+ "darling_macro",
+]
+
+[[package]]
+name = "darling_core"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0c960ae2da4de88a91b2d920c2a7233b400bc33cb28453a2987822d8392519b"
+dependencies = [
+ "fnv",
+ "ident_case",
+ "proc-macro2",
+ "quote",
+ "strsim 0.9.3",
+ "syn",
+]
+
+[[package]]
+name = "darling_macro"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9b5a2f4ac4969822c62224815d069952656cadc7084fdca9751e6d959189b72"
+dependencies = [
+ "darling_core",
+ "quote",
+ "syn",
+]
+
[[package]]
name = "deflate"
version = "1.0.0"
@@ -781,6 +872,31 @@ dependencies = [
"adler32",
]
+[[package]]
+name = "derive_builder"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2658621297f2cf68762a6f7dc0bb7e1ff2cfd6583daef8ee0fed6f7ec468ec0"
+dependencies = [
+ "darling",
+ "derive_builder_core",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "derive_builder_core"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2791ea3e372c8495c0bc2033991d76b512cd799d07491fbd6890124db9458bef"
+dependencies = [
+ "darling",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
[[package]]
name = "derive_more"
version = "0.99.17"
@@ -809,6 +925,25 @@ dependencies = [
"generic-array",
]
+[[package]]
+name = "digest"
+version = "0.10.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f"
+dependencies = [
+ "block-buffer 0.10.3",
+ "crypto-common",
+]
+
+[[package]]
+name = "dirs"
+version = "3.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "30baa043103c9d0c2a57cf537cc2f35623889dc0d405e6c3cccfadbc81c71309"
+dependencies = [
+ "dirs-sys",
+]
+
[[package]]
name = "dirs-next"
version = "2.0.0"
@@ -819,6 +954,17 @@ dependencies = [
"dirs-sys-next",
]
+[[package]]
+name = "dirs-sys"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6"
+dependencies = [
+ "libc",
+ "redox_users",
+ "winapi",
+]
+
[[package]]
name = "dirs-sys-next"
version = "0.1.2"
@@ -881,6 +1027,15 @@ dependencies = [
"cfg-if",
]
+[[package]]
+name = "esaxx-rs"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1f748b253ceca9fed5f42f8b5ceb3851e93102199bc25b64b65369f76e5c0a35"
+dependencies = [
+ "cc",
+]
+
[[package]]
name = "exr"
version = "1.4.2"
@@ -920,7 +1075,7 @@ version = "0.3.0"
source = "git+https://github.com/quickwit-oss/tantivy.git?rev=5d4535de83d214765c29501f0418f1ff38a607b1#5d4535de83d214765c29501f0418f1ff38a607b1"
dependencies = [
"fastdivide",
- "itertools",
+ "itertools 0.10.3",
"log",
"ownedbytes",
"tantivy-bitpacker",
@@ -936,6 +1091,18 @@ dependencies = [
"instant",
]
+[[package]]
+name = "filetime"
+version = "0.2.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4e884668cd0c7480504233e951174ddc3b382f7c2666e3b7310b5c4e7b0c37f9"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "redox_syscall",
+ "windows-sys 0.42.0",
+]
+
[[package]]
name = "fixedbitset"
version = "0.4.2"
@@ -1272,7 +1439,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b"
dependencies = [
"crypto-mac",
- "digest",
+ "digest 0.9.0",
]
[[package]]
@@ -1402,6 +1569,12 @@ dependencies = [
"tokio-native-tls",
]
+[[package]]
+name = "ident_case"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
+
[[package]]
name = "idna"
version = "0.2.3"
@@ -1442,6 +1615,18 @@ dependencies = [
"hashbrown 0.12.1",
]
+[[package]]
+name = "indicatif"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7baab56125e25686df467fe470785512329883aab42696d661247aca2a2896e4"
+dependencies = [
+ "console",
+ "lazy_static",
+ "number_prefix 0.3.0",
+ "regex",
+]
+
[[package]]
name = "indicatif"
version = "0.16.2"
@@ -1450,7 +1635,7 @@ checksum = "2d207dc617c7a380ab07ff572a6e52fa202a2a8f355860ac9c38e23f8196be1b"
dependencies = [
"console",
"lazy_static",
- "number_prefix",
+ "number_prefix 0.4.0",
"rayon",
"regex",
]
@@ -1482,6 +1667,24 @@ version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b"
+[[package]]
+name = "itertools"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f56a2d0bc861f9165be4eb3442afd3c236d8a98afd426f65d92324ae1091a484"
+dependencies = [
+ "either",
+]
+
+[[package]]
+name = "itertools"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b"
+dependencies = [
+ "either",
+]
+
[[package]]
name = "itertools"
version = "0.10.3"
@@ -1553,7 +1756,7 @@ dependencies = [
"bit-set",
"diff",
"ena",
- "itertools",
+ "itertools 0.10.3",
"lalrpop-util",
"petgraph",
"pico-args",
@@ -1717,6 +1920,22 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
+[[package]]
+name = "macro_rules_attribute"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf0c9b980bf4f3a37fd7b1c066941dd1b1d0152ce6ee6e8fe8c49b9f6810d862"
+dependencies = [
+ "macro_rules_attribute-proc_macro",
+ "paste",
+]
+
+[[package]]
+name = "macro_rules_attribute-proc_macro"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "58093314a45e00c77d5c508f76e77c3396afbbc0d01506e7fae47b018bac2b1d"
+
[[package]]
name = "maplit"
version = "1.0.2"
@@ -1758,14 +1977,23 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb"
+[[package]]
+name = "matrixmultiply"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "add85d4dd35074e6fedc608f8c8f513a3548619a9024b751949ef0e8e45a4d84"
+dependencies = [
+ "rawpointer",
+]
+
[[package]]
name = "md-5"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15"
dependencies = [
- "block-buffer",
- "digest",
+ "block-buffer 0.9.0",
+ "digest 0.9.0",
"opaque-debug",
]
@@ -1865,7 +2093,7 @@ dependencies = [
"libc",
"log",
"wasi 0.11.0+wasi-snapshot-preview1",
- "windows-sys",
+ "windows-sys 0.36.1",
]
[[package]]
@@ -1904,6 +2132,19 @@ dependencies = [
"tempfile",
]
+[[package]]
+name = "ndarray"
+version = "0.15.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cc1372704f14bb132a49a6701c2238970a359ee0829fed481b522a63bf25456a"
+dependencies = [
+ "matrixmultiply",
+ "num-complex",
+ "num-integer",
+ "num-traits",
+ "rawpointer",
+]
+
[[package]]
name = "new_debug_unreachable"
version = "1.0.4"
@@ -1926,6 +2167,15 @@ dependencies = [
"minimal-lexical",
]
+[[package]]
+name = "num-complex"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ae39348c8bc5fbd7f40c727a9925f03517afd2ab27d46702108b6a7e5414c19"
+dependencies = [
+ "num-traits",
+]
+
[[package]]
name = "num-integer"
version = "0.1.45"
@@ -1975,6 +2225,12 @@ dependencies = [
"libc",
]
+[[package]]
+name = "number_prefix"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a"
+
[[package]]
name = "number_prefix"
version = "0.4.0"
@@ -1996,6 +2252,51 @@ dependencies = [
"loom",
]
+[[package]]
+name = "onig"
+version = "6.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f"
+dependencies = [
+ "bitflags",
+ "libc",
+ "once_cell",
+ "onig_sys",
+]
+
+[[package]]
+name = "onig_sys"
+version = "69.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b829e3d7e9cc74c7e315ee8edb185bf4190da5acde74afd7fc59c35b1f086e7"
+dependencies = [
+ "cc",
+ "pkg-config",
+]
+
+[[package]]
+name = "onnxruntime"
+version = "0.0.14"
+source = "git+https://github.com/Cuely/onnxruntime-rs#1bf6dedaddf7611a61d79e97105f0571251559fe"
+dependencies = [
+ "lazy_static",
+ "ndarray",
+ "onnxruntime-sys",
+ "thiserror",
+ "tracing",
+]
+
+[[package]]
+name = "onnxruntime-sys"
+version = "0.0.14"
+source = "git+https://github.com/Cuely/onnxruntime-rs#1bf6dedaddf7611a61d79e97105f0571251559fe"
+dependencies = [
+ "flate2",
+ "tar",
+ "ureq",
+ "zip",
+]
+
[[package]]
name = "oorandom"
version = "11.1.3"
@@ -2057,7 +2358,7 @@ dependencies = [
name = "optics"
version = "0.1.0"
dependencies = [
- "itertools",
+ "itertools 0.10.3",
"lalrpop",
"lalrpop-util",
"logos",
@@ -2100,7 +2401,7 @@ dependencies = [
"libc",
"redox_syscall",
"smallvec",
- "windows-sys",
+ "windows-sys 0.36.1",
]
[[package]]
@@ -2109,6 +2410,12 @@ version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd59a245c58efa02bd73c9462fd9d4c28952d650092c6bcba43b60b4707171dd"
+[[package]]
+name = "paste"
+version = "1.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba"
+
[[package]]
name = "peeking_take_while"
version = "0.1.2"
@@ -2444,6 +2751,12 @@ dependencies = [
"rand_core 0.5.1",
]
+[[package]]
+name = "rawpointer"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
+
[[package]]
name = "rayon"
version = "1.5.3"
@@ -2456,6 +2769,17 @@ dependencies = [
"rayon-core",
]
+[[package]]
+name = "rayon-cond"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fd1259362c9065e5ea39a789ef40b1e3fd934c94beb7b5ab3ac6629d3b5e7cb7"
+dependencies = [
+ "either",
+ "itertools 0.8.2",
+ "rayon",
+]
+
[[package]]
name = "rayon-core"
version = "1.9.3"
@@ -2650,7 +2974,7 @@ dependencies = [
"base64",
"bytes",
"chrono",
- "digest",
+ "digest 0.9.0",
"futures",
"hex",
"hmac",
@@ -2663,7 +2987,7 @@ dependencies = [
"rusoto_credential",
"rustc_version",
"serde",
- "sha2",
+ "sha2 0.9.9",
"tokio",
]
@@ -2753,7 +3077,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2"
dependencies = [
"lazy_static",
- "windows-sys",
+ "windows-sys 0.36.1",
]
[[package]]
@@ -2902,13 +3226,24 @@ version = "0.9.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800"
dependencies = [
- "block-buffer",
+ "block-buffer 0.9.0",
"cfg-if",
"cpufeatures",
- "digest",
+ "digest 0.9.0",
"opaque-debug",
]
+[[package]]
+name = "sha2"
+version = "0.10.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "digest 0.10.6",
+]
+
[[package]]
name = "sharded-slab"
version = "0.1.4"
@@ -2976,6 +3311,18 @@ dependencies = [
"lock_api",
]
+[[package]]
+name = "spm_precompiled"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326"
+dependencies = [
+ "base64",
+ "nom",
+ "serde",
+ "unicode-segmentation",
+]
+
[[package]]
name = "stable_deref_trait"
version = "1.2.0"
@@ -3008,6 +3355,18 @@ dependencies = [
"quote",
]
+[[package]]
+name = "strsim"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
+
+[[package]]
+name = "strsim"
+version = "0.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6446ced80d6c486436db5c078dde11a9f73d42b57fb273121e160b84f63d894c"
+
[[package]]
name = "strsim"
version = "0.10.0"
@@ -3057,7 +3416,7 @@ dependencies = [
"fastfield_codecs",
"fs2",
"htmlescape",
- "itertools",
+ "itertools 0.10.3",
"levenshtein_automata",
"log",
"lru",
@@ -3129,6 +3488,17 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
+[[package]]
+name = "tar"
+version = "0.4.38"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b55807c0344e1e6c04d7c965f5289c39a8d94ae23ed5c0b57aabac549f871c6"
+dependencies = [
+ "filetime",
+ "libc",
+ "xattr",
+]
+
[[package]]
name = "tempfile"
version = "3.3.0"
@@ -3311,6 +3681,41 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
+[[package]]
+name = "tokenizers"
+version = "0.13.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f4ff2dd291eac98dcea13e8cf7a0b28c373a90dc9210ccdab0fa9e69ee0cac69"
+dependencies = [
+ "aho-corasick",
+ "cached-path",
+ "clap 2.34.0",
+ "derive_builder",
+ "dirs",
+ "esaxx-rs",
+ "getrandom 0.2.6",
+ "indicatif 0.15.0",
+ "itertools 0.9.0",
+ "lazy_static",
+ "log",
+ "macro_rules_attribute",
+ "onig",
+ "paste",
+ "rand 0.8.5",
+ "rayon",
+ "rayon-cond",
+ "regex",
+ "regex-syntax",
+ "reqwest",
+ "serde",
+ "serde_json",
+ "spm_precompiled",
+ "thiserror",
+ "unicode-normalization-alignments",
+ "unicode-segmentation",
+ "unicode_categories",
+]
+
[[package]]
name = "tokio"
version = "1.19.2"
@@ -3559,6 +3964,21 @@ dependencies = [
"tinyvec",
]
+[[package]]
+name = "unicode-normalization-alignments"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de"
+dependencies = [
+ "smallvec",
+]
+
+[[package]]
+name = "unicode-segmentation"
+version = "1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fdbf052a0783de01e944a6ce7a8cb939e295b1e7be835a1112c3b9a7f047a5a"
+
[[package]]
name = "unicode-width"
version = "0.1.9"
@@ -3571,12 +3991,35 @@ version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c"
+[[package]]
+name = "unicode_categories"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
+
[[package]]
name = "untrusted"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
+[[package]]
+name = "ureq"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b97acb4c28a254fd7a4aeec976c46a7fa404eac4d7c134b30c75144846d7cb8f"
+dependencies = [
+ "base64",
+ "chunked_transfer",
+ "flate2",
+ "log",
+ "once_cell",
+ "rustls",
+ "url",
+ "webpki",
+ "webpki-roots",
+]
+
[[package]]
name = "url"
version = "2.2.2"
@@ -3635,6 +4078,12 @@ version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
+[[package]]
+name = "vec_map"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
+
[[package]]
name = "version_check"
version = "0.9.4"
@@ -3766,6 +4215,15 @@ dependencies = [
"untrusted",
]
+[[package]]
+name = "webpki-roots"
+version = "0.22.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87"
+dependencies = [
+ "webpki",
+]
+
[[package]]
name = "weezl"
version = "0.1.7"
@@ -3819,43 +4277,100 @@ version = "0.36.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2"
dependencies = [
- "windows_aarch64_msvc",
- "windows_i686_gnu",
- "windows_i686_msvc",
- "windows_x86_64_gnu",
- "windows_x86_64_msvc",
+ "windows_aarch64_msvc 0.36.1",
+ "windows_i686_gnu 0.36.1",
+ "windows_i686_msvc 0.36.1",
+ "windows_x86_64_gnu 0.36.1",
+ "windows_x86_64_msvc 0.36.1",
]
+[[package]]
+name = "windows-sys"
+version = "0.42.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7"
+dependencies = [
+ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc 0.42.0",
+ "windows_i686_gnu 0.42.0",
+ "windows_i686_msvc 0.42.0",
+ "windows_x86_64_gnu 0.42.0",
+ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc 0.42.0",
+]
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.42.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e"
+
[[package]]
name = "windows_aarch64_msvc"
version = "0.36.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47"
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.42.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4"
+
[[package]]
name = "windows_i686_gnu"
version = "0.36.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6"
+[[package]]
+name = "windows_i686_gnu"
+version = "0.42.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7"
+
[[package]]
name = "windows_i686_msvc"
version = "0.36.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024"
+[[package]]
+name = "windows_i686_msvc"
+version = "0.42.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246"
+
[[package]]
name = "windows_x86_64_gnu"
version = "0.36.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1"
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.42.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.42.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028"
+
[[package]]
name = "windows_x86_64_msvc"
version = "0.36.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680"
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.42.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5"
+
[[package]]
name = "winreg"
version = "0.10.1"
@@ -3874,6 +4389,15 @@ dependencies = [
"tap",
]
+[[package]]
+name = "xattr"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d1526bbe5aaeb5eb06885f4d987bcdfa5e23187055de9b83fe00156a821fabc"
+dependencies = [
+ "libc",
+]
+
[[package]]
name = "xml-rs"
version = "0.8.4"
@@ -3886,6 +4410,29 @@ version = "1.5.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c394b5bd0c6f669e7275d9c20aa90ae064cb22e75a1cad54e1b34088034b149f"
+[[package]]
+name = "zip"
+version = "0.5.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93ab48844d61251bb3835145c521d88aa4031d7139e8485990f60ca911fa0815"
+dependencies = [
+ "byteorder",
+ "bzip2",
+ "crc32fast",
+ "flate2",
+ "thiserror",
+ "time 0.1.44",
+]
+
+[[package]]
+name = "zip-extensions"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a64c3c977bc3434ce2d4bcea8ad3c644672de0f2c402b72b9171ca80a8885d14"
+dependencies = [
+ "zip",
+]
+
[[package]]
name = "zstd-sys"
version = "2.0.1+zstd.1.5.2"
diff --git a/Cargo.toml b/Cargo.toml
index 684df09b..e32ec6d0 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -64,4 +64,6 @@ criterion = "0.3.6"
maplit = "1.0.2"
rand = "0.8.5"
crossbeam-channel = "0.5.6"
-num_cpus = "1.15.0"
\ No newline at end of file
+num_cpus = "1.15.0"
+onnxruntime = { git = "https://github.com/Cuely/onnxruntime-rs" }
+tokenizers = "0.13.2"
\ No newline at end of file
diff --git a/core/Cargo.toml b/core/Cargo.toml
index 6586779f..34382625 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -69,6 +69,8 @@ once_cell = { workspace = true }
optics = { path = "../optics" }
crossbeam-channel = { workspace = true }
num_cpus = { workspace = true }
+onnxruntime = { workspace = true }
+tokenizers = { workspace = true }
[dev-dependencies]
criterion = { workspace = true }
diff --git a/core/examples/print_approx_harmonic_centrality.rs b/core/examples/print_approx_harmonic_centrality.rs
index b11d723e..28119ebc 100644
--- a/core/examples/print_approx_harmonic_centrality.rs
+++ b/core/examples/print_approx_harmonic_centrality.rs
@@ -96,10 +96,7 @@ fn print_top_nodes(
let scorer = store.online_harmonic.scorer(&liked_nodes, &[]);
let top_nodes = get_top_nodes(scorer, top_n, nodes);
- println!(
- "top {} sites for these liked sites {:?}",
- top_n, liked_sites
- );
+ println!("top {top_n} sites for these liked sites {liked_sites:?}");
print_nodes(&top_nodes, id2node);
println!();
diff --git a/core/examples/print_inbound_links.rs b/core/examples/print_inbound_links.rs
index 0b11f933..e44dcbba 100644
--- a/core/examples/print_inbound_links.rs
+++ b/core/examples/print_inbound_links.rs
@@ -25,7 +25,7 @@ pub fn main() {
"onlinester.ga",
"GroomersNew.ga",
] {
- println!("{}:", host);
+ println!("{host}:");
for edge in graph.ingoing_edges(Node::from(host)) {
println!("{} -> {} ({})", edge.from.name, edge.to.name, edge.label);
}
diff --git a/core/src/entrypoint/frontend.rs b/core/src/entrypoint/frontend.rs
index a7137f6e..64f633dd 100644
--- a/core/src/entrypoint/frontend.rs
+++ b/core/src/entrypoint/frontend.rs
@@ -18,8 +18,13 @@ use anyhow::Result;
use crate::frontend::router;
-pub async fn run(queries_csv_path: &str, host: &str, shards: Vec<Vec<String>>) -> Result<()> {
- let app = router(queries_csv_path, shards)?;
+pub async fn run(
+ queries_csv_path: &str,
+ crossencoder_model_path: &str,
+ host: &str,
+ shards: Vec<Vec<String>>,
+) -> Result<()> {
+ let app = router(queries_csv_path, crossencoder_model_path, shards)?;
let addr = host.parse()?;
tracing::info!("listening on {}", addr);
axum::Server::bind(&addr)
diff --git a/core/src/frontend/mod.rs b/core/src/frontend/mod.rs
index bcb58874..66f9891d 100644
--- a/core/src/frontend/mod.rs
+++ b/core/src/frontend/mod.rs
@@ -19,6 +19,7 @@ use tower_http::{compression::CompressionLayer, services::ServeDir};
use crate::{
autosuggest::Autosuggest,
+ ranking::models::cross_encoder::CrossEncoderModel,
searcher::{DistributedSearcher, Shard},
};
use anyhow::Result;
@@ -57,7 +58,7 @@ where
Ok(html) => Html(html).into_response(),
Err(err) => (
StatusCode::INTERNAL_SERVER_ERROR,
- format!("Failed to render template. Error: {}", err),
+ format!("Failed to render template. Error: {err}"),
)
.into_response(),
}
@@ -74,7 +75,11 @@ pub async fn favicon() -> impl IntoResponse {
.unwrap()
}
-pub fn router(queries_csv_path: &str, shards: Vec<Vec<String>>) -> Result<Router> {
+pub fn router(
+ queries_csv_path: &str,
+ crossencoder_model_path: &str,
+ shards: Vec<Vec<String>>,
+) -> Result<Router> {
let shards: Vec<_> = shards
.into_iter()
.enumerate()
@@ -82,7 +87,8 @@ pub fn router(queries_csv_path: &str, shards: Vec<Vec<String>>) -> Result<Router
.collect();
let autosuggest = Autosuggest::load_csv(queries_csv_path)?;
- let searcher = DistributedSearcher::new(shards);
+ let crossencoder = CrossEncoderModel::open(crossencoder_model_path)?;
+ let searcher = DistributedSearcher::new(shards, crossencoder);
let state = Arc::new(State {
searcher,
@@ -106,7 +112,7 @@ pub fn router(queries_csv_path: &str, shards: Vec<Vec<String>>) -> Result<Router
|error: std::io::Error| async move {
(
StatusCode::INTERNAL_SERVER_ERROR,
- format!("Unhandled internal error: {}", error),
+ format!("Unhandled internal error: {error}"),
)
},
))
diff --git a/core/src/hyperloglog.rs b/core/src/hyperloglog.rs
index 7f1f0b61..9ecc8f75 100644
--- a/core/src/hyperloglog.rs
+++ b/core/src/hyperloglog.rs
@@ -4038,13 +4038,13 @@ use std::array;
#[derive(Clone)]
pub struct HyperLogLog<const N: usize> {
- registers: [u8; N],
+ registers: Box<[u8; N]>,
}
impl<const N: usize> Default for HyperLogLog<N> {
fn default() -> Self {
Self {
- registers: array::from_fn(|_| 0),
+ registers: Box::new(array::from_fn(|_| 0)),
}
}
}
@@ -4202,7 +4202,7 @@ impl<const N: usize> HyperLogLog<N> {
}
pub fn registers(&self) -> &[u8] {
- &self.registers
+ self.registers.as_ref()
}
}
diff --git a/core/src/image_store.rs b/core/src/image_store.rs
index d817417b..c3bb50b7 100644
--- a/core/src/image_store.rs
+++ b/core/src/image_store.rs
@@ -20,7 +20,7 @@ use image::imageops::FilterType;
use image::{DynamicImage, ImageOutputFormat};
use serde::{de, ser::SerializeStruct, Serialize};
use std::collections::HashSet;
-use std::io::{Cursor, Read, Seek, SeekFrom};
+use std::io::{Cursor, Read, Seek};
use std::path::Path;
use uuid::Uuid;
@@ -295,7 +295,7 @@ impl Image {
self.0
.write_to(&mut cursor, ImageOutputFormat::Png)
.unwrap();
- cursor.seek(SeekFrom::Start(0)).unwrap();
+ cursor.rewind().unwrap();
let mut bytes = Vec::new();
cursor.read_to_end(&mut bytes).unwrap();
diff --git a/core/src/kv/rocksdb_store.rs b/core/src/kv/rocksdb_store.rs
index d2efae6f..655feed2 100644
--- a/core/src/kv/rocksdb_store.rs
+++ b/core/src/kv/rocksdb_store.rs
@@ -82,7 +82,7 @@ where
if let Err(err) = self.flush() {
match err.kind() {
rocksdb::ErrorKind::NotSupported => {}
- _ => panic!("failed to flush: {:?}", err),
+ _ => panic!("failed to flush: {err:?}"),
}
}
}
diff --git a/core/src/lib.rs b/core/src/lib.rs
index 389190c2..bf9f48a4 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -151,6 +151,7 @@ pub struct HttpConfig {
pub struct FrontendConfig {
pub queries_csv_path: String,
pub host: String,
+ pub crossencoder_model_path: String,
pub search_servers: Vec<Vec<String>>,
}
@@ -242,6 +243,12 @@ pub enum Error {
#[error("Internal error")]
InternalError(String),
+
+ #[error("Huggingface tokenizer")]
+ HuggingfaceTokenizer(#[from] tokenizers::Error),
+
+ #[error("Onnx")]
+ Onnx(#[from] onnxruntime::OrtError),
}
pub(crate) type Result<T> = std::result::Result<T, Error>;
diff --git a/core/src/main.rs b/core/src/main.rs
index f09e84c9..62fe25cb 100644
--- a/core/src/main.rs
+++ b/core/src/main.rs
@@ -190,6 +190,7 @@ fn main() -> Result<()> {
.build()?
.block_on(frontend::run(
&config.queries_csv_path,
+ &config.crossencoder_model_path,
&config.host,
config.search_servers,
))?
diff --git a/core/src/mapreduce/manager.rs b/core/src/mapreduce/manager.rs
index ea322173..d2f81d67 100644
--- a/core/src/mapreduce/manager.rs
+++ b/core/src/mapreduce/manager.rs
@@ -115,7 +115,7 @@ impl WorkerPool {
.iter()
.flat_map(|addr| {
addr.to_socket_addrs().unwrap_or_else(|_| {
- panic!("failed to transform {:?} into a socket address", addr)
+ panic!("failed to transform {addr:?} into a socket address")
})
})
.map(|addr| Arc::new(RemoteWorker { addr }))
diff --git a/core/src/query/pattern_query.rs b/core/src/query/pattern_query.rs
index 52d2fc45..2fca2ddb 100644
--- a/core/src/query/pattern_query.rs
+++ b/core/src/query/pattern_query.rs
@@ -222,15 +222,13 @@ impl tantivy::query::Weight for PatternWeight {
let scorer_opt = self.pattern_scorer(reader, 1.0)?;
if scorer_opt.is_none() {
return Err(TantivyError::InvalidArgument(format!(
- "Document #({}) does not match",
- doc
+ "Document #({doc}) does not match"
)));
}
let mut scorer = scorer_opt.unwrap();
if scorer.seek(doc) != doc {
return Err(TantivyError::InvalidArgument(format!(
- "Document #({}) does not match",
- doc
+ "Document #({doc}) does not match"
)));
}
let fieldnorm_reader = self.fieldnorm_reader(reader)?;
diff --git a/core/src/ranking/bm25.rs b/core/src/ranking/bm25.rs
index 7c45a5b9..e79de857 100644
--- a/core/src/ranking/bm25.rs
+++ b/core/src/ranking/bm25.rs
@@ -10,7 +10,7 @@ const K1: Score = 1.2;
const B: Score = 0.75;
pub(crate) fn idf(doc_freq: u64, doc_count: u64) -> Score {
- assert!(doc_count >= doc_freq, "{} >= {}", doc_count, doc_freq);
+ assert!(doc_count >= doc_freq, "{doc_count} >= {doc_freq}");
let x = ((doc_count - doc_freq) as Score + 0.5) / (doc_freq as Score + 0.5);
(1.0 + x).ln()
}
diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rs
index a6d3bea2..a52b2aa1 100644
--- a/core/src/ranking/mod.rs
+++ b/core/src/ranking/mod.rs
@@ -17,6 +17,7 @@
pub mod bm25;
pub mod centrality_store;
pub mod initial;
+pub mod models;
pub mod optics;
pub mod pipeline;
pub mod signal;
diff --git a/core/src/ranking/models/cross_encoder.rs b/core/src/ranking/models/cross_encoder.rs
new file mode 100644
index 00000000..9ea8f7a2
--- /dev/null
+++ b/core/src/ranking/models/cross_encoder.rs
@@ -0,0 +1,123 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::{path::Path, sync::Mutex};
+
+use itertools::Itertools;
+use onnxruntime::{ndarray::ArrayBase, tensor::OrtOwnedTensor, GraphOptimizationLevel};
+
+use crate::Result;
+
+use super::ONNX_ENVIRONMENT;
+
+pub struct CrossEncoderModel {
+ tokenizer: tokenizers::Tokenizer,
+ session: Mutex<onnxruntime::session::Session<'static>>,
+}
+
+impl CrossEncoderModel {
+ pub fn open<P: AsRef<Path>>(folder: P) -> Result<Self> {
+ let tokenizer = tokenizers::Tokenizer::from_file(folder.as_ref().join("tokenizer.json"))?;
+
+ let session = Mutex::new(
+ ONNX_ENVIRONMENT
+ .new_session_builder()?
+ .with_optimization_level(GraphOptimizationLevel::All)?
+ .with_number_threads(1)?
+ .with_model_from_file(folder.as_ref().join("model.onnx"))?,
+ );
+
+ Ok(Self { tokenizer, session })
+ }
+}
+
+pub trait CrossEncoder: Send + Sync {
+ fn run(&self, query: &str, body: &str) -> f64;
+}
+
+pub struct DummyCrossEncoder {}
+
+impl CrossEncoder for DummyCrossEncoder {
+ fn run(&self, _query: &str, _body: &str) -> f64 {
+ 1.0
+ }
+}
+
+impl CrossEncoder for CrossEncoderModel {
+ fn run(&self, query: &str, body: &str) -> f64 {
+ let encoded = self.tokenizer.encode((query, body), true).unwrap();
+
+ let ids = encoded.get_ids().iter().map(|i| *i as i64).collect_vec();
+
+ let attention_mask = encoded
+ .get_attention_mask()
+ .iter()
+ .map(|i| *i as i64)
+ .collect_vec();
+
+ let type_ids = encoded
+ .get_type_ids()
+ .iter()
+ .map(|i| *i as i64)
+ .collect_vec();
+
+ let num_tokens = ids.len();
+
+ let mut sess = self.session.lock().unwrap();
+
+ let res: Vec<OrtOwnedTensor<f64, _>> = sess
+ .run(vec![
+ ArrayBase::from_vec(ids)
+ .into_shape((1, num_tokens))
+ .unwrap(),
+ ArrayBase::from_vec(attention_mask)
+ .into_shape((1, num_tokens))
+ .unwrap(),
+ ArrayBase::from_vec(type_ids)
+ .into_shape((1, num_tokens))
+ .unwrap(),
+ ])
+ .unwrap();
+
+ res[0][[0, 0]]
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn sanity_check() {
+ match CrossEncoderModel::open("../data/cross_encoder") {
+ Ok(model) => {
+ assert!(
+ model.run(
+ "how many people live in paris",
+ "there are currently 1234 people living in paris"
+ ) > model.run(
+ "how many people live in paris",
+ "I really like cake and my favorite cake is probably brownie"
+ )
+ );
+ }
+ Err(err) => {
+ dbg!(err);
+ // panic!();
+ }
+ }
+ }
+}
diff --git a/core/src/ranking/models/mod.rs b/core/src/ranking/models/mod.rs
new file mode 100644
index 00000000..13e94d77
--- /dev/null
+++ b/core/src/ranking/models/mod.rs
@@ -0,0 +1,28 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use onnxruntime::{environment::Environment, LoggingLevel};
+
+pub mod cross_encoder;
+
+pub static ONNX_ENVIRONMENT: once_cell::sync::Lazy<Environment> =
+ once_cell::sync::Lazy::new(|| {
+ Environment::builder()
+ .with_name("ranking")
+ .with_log_level(LoggingLevel::Info)
+ .build()
+ .unwrap()
+ });
diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rs
index f35e9751..2d081fdb 100644
--- a/core/src/ranking/pipeline.rs
+++ b/core/src/ranking/pipeline.rs
@@ -14,17 +14,19 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::cmp::Ordering;
+use std::{cmp::Ordering, sync::Arc};
+use optics::ast::{RankingCoeff, RankingTarget};
use serde::{Deserialize, Serialize};
use crate::{
inverted_index::WebsitePointer,
schema::{FastField, Field, TextField, ALL_FIELDS, FLOAT_SCALING},
searcher::SearchQuery,
+ Result,
};
-use super::SignalAggregator;
+use super::{models::cross_encoder::CrossEncoder, SignalAggregator};
pub trait AsRankingWebsite: Clone {
fn as_ranking(&self) -> &RankingWebsite;
@@ -105,26 +107,51 @@ impl RankingWebsite {
trait Scorer<T: AsRankingWebsite>: Send + Sync {
fn score(&self, websites: &mut [T]);
+ fn set_coefficients(&mut self, coefficients: &[RankingCoeff]);
+ fn set_query_info(&mut self, _query: &SearchQuery) {}
}
-struct ReRanker {
- crossencoder: f64,
+struct ReRanker<M: CrossEncoder> {
+ coefficient: f64,
+ model: Arc<M>,
prev_score: f64,
+ query: String,
}
-impl Default for ReRanker {
- fn default() -> Self {
+impl<M: CrossEncoder> ReRanker<M> {
+ fn new(model: Arc<M>) -> Self {
Self {
- crossencoder: 1.0,
+ coefficient: 1.0,
+ model,
prev_score: 1.0,
+ query: String::new(),
}
}
}
-impl<T: AsRankingWebsite> Scorer<T> for ReRanker {
+impl<T: AsRankingWebsite, M: CrossEncoder> Scorer<T> for ReRanker<M> {
fn score(&self, websites: &mut [T]) {
- // TODO: Implement actual scoring
- // todo!();
+ for website in websites {
+ let mut website = website.as_mut_ranking();
+ website.score = self.prev_score * website.score
+ + self.coefficient * self.model.run(&self.query, &website.clean_body);
+ }
+ }
+
+ fn set_query_info(&mut self, query: &SearchQuery) {
+ self.query = query.query.clone();
+ }
+
+ fn set_coefficients(&mut self, coefficients: &[RankingCoeff]) {
+ for coeff in coefficients {
+ if let RankingTarget::Signal(signal) = &coeff.target {
+ match signal.as_str() {
+ "prev_score" => self.prev_score = coeff.score,
+ "crossencoder" => self.coefficient = coeff.score,
+ _ => {}
+ }
+ }
+ }
}
}
@@ -150,6 +177,18 @@ impl<T: AsRankingWebsite> Scorer<T> for PrioritizeText {
website.as_mut_ranking().score = self.bm25 * bm25 + self.prev_score * prev_score;
}
}
+
+ fn set_coefficients(&mut self, coefficients: &[RankingCoeff]) {
+ for coeff in coefficients {
+ if let RankingTarget::Signal(signal) = &coeff.target {
+ match signal.as_str() {
+ "prev_score" => self.prev_score = coeff.score,
+ "bm25" => self.bm25 = coeff.score,
+ _ => {}
+ }
+ }
+ }
+ }
}
enum Prev<T: AsRankingWebsite> {
@@ -222,6 +261,14 @@ impl<T: AsRankingWebsite> RankingStage<T> {
.take(top_n)
.collect()
}
+
+ fn set_query_info(&mut self, query: &SearchQuery) {
+ self.scorer.set_query_info(query);
+ match &mut self.prev {
+ Prev::Initial => {}
+ Prev::Node(prev) => prev.set_query_info(query),
+ }
+ }
}
pub struct RankingPipeline<T: AsRankingWebsite> {
@@ -231,9 +278,9 @@ pub struct RankingPipeline<T: AsRankingWebsite> {
}
impl<T: AsRankingWebsite> RankingPipeline<T> {
- fn create() -> Self {
+ fn create<M: CrossEncoder + 'static>(crossencoder: Arc<M>) -> Result<Self> {
let last_stage = RankingStage {
- scorer: Box::<ReRanker>::default(),
+ scorer: Box::new(ReRanker::new(crossencoder)),
prev: Prev::Node(Box::new(RankingStage {
scorer: Box::<PrioritizeText>::default(),
prev: Prev::Initial,
@@ -244,17 +291,43 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {
top_n: 50,
};
- Self {
+ Ok(Self {
last_stage,
offset: 0,
top_n: 0,
- }
+ })
}
- pub fn for_query(query: &mut SearchQuery) -> Self {
- dbg!(&query.optic_program);
+ pub fn for_query<M: CrossEncoder + 'static>(
+ query: &mut SearchQuery,
+ crossencoder: Arc<M>,
+ ) -> Result<Self> {
+ let mut pipeline = Self::create(crossencoder)?;
+ if let Some(optic) = &query.optic_program {
+ let optic = optics::parse(optic)?;
+ if let Some(optic_pipeline) = optic.pipeline {
+ let mut stage = Some(&mut pipeline.last_stage);
+
+ // skip(1) since first stage is collector
+ for optic_stage in optic_pipeline.stages.into_iter().skip(1).rev() {
+ match stage {
+ Some(inner_stage) => {
+ inner_stage
+ .scorer
+ .set_coefficients(&optic_stage.coefficients);
+
+ match &mut inner_stage.prev {
+ Prev::Initial => stage = None,
+ Prev::Node(next) => stage = Some(next),
+ }
+ }
+ None => break,
+ }
+ }
+ }
+ }
- let mut pipeline = Self::create();
+ pipeline.last_stage.set_query_info(query);
pipeline.offset = query.offset;
pipeline.top_n = query.num_results;
@@ -262,7 +335,7 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {
query.num_results = pipeline.collector_top_n();
query.offset = pipeline.collector_offset();
- pipeline
+ Ok(pipeline)
}
pub fn apply(mut self, websites: Vec<T>) -> Vec<T> {
@@ -288,6 +361,7 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {
mod tests {
use itertools::Itertools;
+ use crate::ranking::models::cross_encoder::DummyCrossEncoder;
use crate::{
collector::Hashes, inverted_index::DocAddress, prehashed::Prehashed,
ranking::initial::Score,
@@ -330,9 +404,13 @@ mod tests {
#[test]
fn simple() {
- let pipeline = RankingPipeline::for_query(&mut SearchQuery {
- ..Default::default()
- });
+ let pipeline = RankingPipeline::for_query(
+ &mut SearchQuery {
+ ..Default::default()
+ },
+ Arc::new(DummyCrossEncoder {}),
+ )
+ .unwrap();
assert_eq!(pipeline.collector_top_n(), 200);
let sample = sample_websites(pipeline.collector_top_n());
@@ -353,10 +431,14 @@ mod tests {
#[test]
fn offsets() {
- let pipeline = RankingPipeline::for_query(&mut SearchQuery {
- offset: 0,
- ..Default::default()
- });
+ let pipeline = RankingPipeline::for_query(
+ &mut SearchQuery {
+ offset: 0,
+ ..Default::default()
+ },
+ Arc::new(DummyCrossEncoder {}),
+ )
+ .unwrap();
let sample: Vec<_> =
sample_websites(pipeline.collector_top_n() + pipeline.collector_offset())
@@ -366,10 +448,14 @@ mod tests {
let mut prev: Vec<_> = pipeline.apply(sample);
for offset in 1..1_000 {
- let pipeline = RankingPipeline::for_query(&mut SearchQuery {
- offset,
- ..Default::default()
- });
+ let pipeline = RankingPipeline::for_query(
+ &mut SearchQuery {
+ offset,
+ ..Default::default()
+ },
+ Arc::new(DummyCrossEncoder {}),
+ )
+ .unwrap();
let sample: Vec<_> =
sample_websites(pipeline.collector_top_n() + pipeline.collector_offset())
@@ -386,24 +472,18 @@ mod tests {
.any(|r| r.pointer.address.doc_id == first.pointer.address.doc_id));
}
- // assert_eq!(
- // prev.iter()
- // .map(|p| usize::from(
- // res.iter()
- // .any(|r| r.pointer.address.doc_id == p.pointer.address.doc_id)
- // ))
- // .sum::<usize>(),
- // 19,
- // "Only the top result from previous offset should be removed in current ranking"
- // );
-
prev = res;
}
- let pipeline = RankingPipeline::for_query(&mut SearchQuery {
- offset: 0,
- ..Default::default()
- });
+ let pipeline = RankingPipeline::for_query(
+ &mut SearchQuery {
+ offset: 0,
+ ..Default::default()
+ },
+ Arc::new(DummyCrossEncoder {}),
+ )
+ .unwrap();
+
let sample: Vec<_> =
sample_websites(pipeline.collector_top_n() + pipeline.collector_offset())
.into_iter()
@@ -411,10 +491,14 @@ mod tests {
.collect();
let mut prev: Vec<_> = pipeline.apply(sample);
for p in 1..100 {
- let pipeline = RankingPipeline::for_query(&mut SearchQuery {
- offset: p * 20,
- ..Default::default()
- });
+ let pipeline = RankingPipeline::for_query(
+ &mut SearchQuery {
+ offset: p * 20,
+ ..Default::default()
+ },
+ Arc::new(DummyCrossEncoder {}),
+ )
+ .unwrap();
let sample: Vec<_> =
sample_websites(pipeline.collector_top_n() + pipeline.collector_offset())
@@ -436,29 +520,29 @@ mod tests {
#[test]
fn multistage_coefficients() {
- let pipeline = RankingPipeline::for_query(&mut SearchQuery {
- optic_program: Some(
- r#"
+ let pipeline = RankingPipeline::for_query(
+ &mut SearchQuery {
+ optic_program: Some(
+ r#"
RankingPipeline {
Stage {},
Stage {
- Ranking {
- Signal("bm25", 3),
- Signal("prev_score", 2),
- },
+ Ranking{Signal("bm25"), 3},
+ Ranking{Signal("prev_score"), 2},
},
Stage {
- Ranking {
- Signal("crossencoder", 4),
- Signal("prev_score", 3),
- },
+ Ranking{Signal("crossencoder"), 4},
+ Ranking{Signal("prev_score"), 3},
},
}
"#
- .to_string(),
- ),
- ..Default::default()
- });
+ .to_string(),
+ ),
+ ..Default::default()
+ },
+ Arc::new(DummyCrossEncoder {}),
+ )
+ .unwrap();
let w = RankingWebsite {
pointer: WebsitePointer {
@@ -487,16 +571,43 @@ mod tests {
score: 1.0,
};
+ let res = pipeline.apply(vec![w.clone()]);
+ assert_eq!(res[0].score, 19.0);
+
+ let pipeline = RankingPipeline::for_query(
+ &mut SearchQuery {
+ optic_program: Some(
+ r#"
+ RankingPipeline {
+ Stage {},
+ Stage {
+ Ranking{Signal("bm25"), 3},
+ Ranking{Signal("prev_score"), 2},
+ },
+ Stage {
+ Ranking{Signal("crossencoder"), 4},
+ Ranking{Signal("prev_score"), 3},
+ },
+ }
+ "#
+ .to_string(),
+ ),
+ ..Default::default()
+ },
+ Arc::new(DummyCrossEncoder {}),
+ )
+ .unwrap();
+
let mut test = [w.clone()];
pipeline.last_stage.scorer.score(&mut test);
- assert_eq!(test[0].score, 5.0);
+ assert_eq!(test[0].score, 7.0);
let mut test = [w];
if let Prev::Node(prev) = pipeline.last_stage.prev {
prev.scorer.score(&mut test);
- assert_eq!(test[0].score, 3.0);
+ assert_eq!(test[0].score, 5.0);
}
}
}
diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rs
index 3c80f5cf..786d545b 100644
--- a/core/src/searcher/distributed.rs
+++ b/core/src/searcher/distributed.rs
@@ -18,12 +18,15 @@ use crate::{
collector::{self, BucketCollector},
exponential_backoff::ExponentialBackoff,
inverted_index::{self},
- ranking::pipeline::{AsRankingWebsite, RankingPipeline, RankingWebsite},
+ ranking::{
+ models::cross_encoder::CrossEncoderModel,
+ pipeline::{AsRankingWebsite, RankingPipeline, RankingWebsite},
+ },
search_prettifier::DisplayedWebpage,
searcher::PrettifiedWebsitesResult,
};
-use std::{net::SocketAddr, time::Instant};
+use std::{net::SocketAddr, sync::Arc, time::Instant};
use futures::stream::FuturesUnordered;
use futures::StreamExt;
@@ -47,6 +50,9 @@ pub enum Error {
#[error("Query cannot be empty")]
EmptyQuery,
+
+ #[error("Internal error")]
+ InternalError(#[from] crate::Error),
}
impl RemoteSearcher {
@@ -171,6 +177,7 @@ pub enum Request {
pub struct DistributedSearcher {
shards: Vec<Shard>,
+ cross_encoder: Arc<CrossEncoderModel>,
}
#[derive(Clone)]
@@ -204,8 +211,11 @@ impl collector::Doc for ScoredWebsitePointer {
}
impl DistributedSearcher {
- pub fn new(shards: Vec<Shard>) -> Self {
- Self { shards }
+ pub fn new(shards: Vec<Shard>, model: CrossEncoderModel) -> Self {
+ Self {
+ shards,
+ cross_encoder: Arc::new(model),
+ }
}
pub async fn search(&self, query: &SearchQuery) -> Result<PrettifiedSearchResult> {
@@ -217,7 +227,7 @@ impl DistributedSearcher {
let mut search_query = query.clone();
let pipeline: RankingPipeline<ScoredWebsitePointer> =
- RankingPipeline::for_query(&mut search_query);
+ RankingPipeline::for_query(&mut search_query, self.cross_encoder.clone())?;
// search shards
let initial_results = self
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index deca8413..05dfb3f0 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -15,7 +15,6 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use std::str::FromStr;
-use std::time::Instant;
use optics::Optic;
use serde::{Deserialize, Serialize};
@@ -28,7 +27,7 @@ use crate::index::Index;
use crate::query::Query;
use crate::ranking::centrality_store::CentralityStore;
use crate::ranking::optics::CreateAggregator;
-use crate::ranking::pipeline::{RankingPipeline, RankingWebsite};
+use crate::ranking::pipeline::RankingWebsite;
use crate::ranking::{online_centrality_scorer, Ranker, SignalAggregator};
use crate::spell::Correction;
use crate::webgraph::centrality::topic::TopicCentrality;
@@ -36,7 +35,10 @@ use crate::webpage::region::Region;
use crate::webpage::Url;
use crate::{inverted_index, Error, Result};
-use super::{InitialSearchResult, SearchQuery, SearchResult, Sidebar, WebsitesResult};
+use super::{InitialSearchResult, SearchQuery, SearchResult, Sidebar};
+
+#[cfg(test)]
+use super::WebsitesResult;
const STACKOVERFLOW_SIDEBAR_SCORE_THRESHOLD: f64 = 250.0;
@@ -309,12 +311,24 @@ impl LocalSearcher {
self.index.retrieve_websites(websites, &query)
}
+ #[cfg(test)]
pub fn search(&self, query: &SearchQuery) -> Result<SearchResult> {
- let start = Instant::now();
+ use std::{sync::Arc, time::Instant};
+ use crate::ranking::{
+ models::cross_encoder::{CrossEncoderModel, DummyCrossEncoder},
+ pipeline::RankingPipeline,
+ };
+
+ let start = Instant::now();
let mut search_query = query.clone();
- let pipeline = RankingPipeline::for_query(&mut search_query);
+ let pipeline = match CrossEncoderModel::open("data/cross_encoder") {
+ Ok(model) => RankingPipeline::for_query(&mut search_query, Arc::new(model))?,
+ Err(_) => {
+ RankingPipeline::for_query(&mut search_query, Arc::new(DummyCrossEncoder {}))?
+ }
+ };
let initial_result = self.search_initial(&search_query, true)?;
diff --git a/core/src/snippet.rs b/core/src/snippet.rs
index e292cc3c..e4c6bf6c 100644
--- a/core/src/snippet.rs
+++ b/core/src/snippet.rs
@@ -127,9 +127,9 @@ impl Snippet {
///
/// Fragments must be valid in the sense that `&text[fragment.start..fragment.stop]`\
/// has to be a valid string.
-fn search_fragments<'a>(
+fn search_fragments(
tokenizer: &TextAnalyzer,
- text: &'a str,
+ text: &str,
terms: &BTreeMap<String, Score>,
max_num_chars: usize,
) -> Vec<FragmentCandidate> {
@@ -373,11 +373,10 @@ Survey in 2016, 2017, and 2018."#;
<title>Website for runners</title>
</head>
<body>
- {}
+ {TEST_TEXT}
</body>
</html>
- "#,
- TEST_TEXT
+ "#
),
"https://www.example.com",
))
@@ -413,11 +412,10 @@ Survey in 2016, 2017, and 2018."#;
<title>Website for runners</title>
</head>
<body>
- {}
+ {TEST_TEXT}
</body>
</html>
- "#,
- TEST_TEXT
+ "#
),
"https://www.example.com",
))
@@ -623,11 +621,10 @@ Survey in 2016, 2017, and 2018."#;
<title>Website for runners</title>
</head>
<body>
- {}
+ {TEST_TEXT}
</body>
</html>
- "#,
- TEST_TEXT
+ "#
),
"https://www.example.com",
))
diff --git a/core/src/warc.rs b/core/src/warc.rs
index a709178c..7bee7682 100644
--- a/core/src/warc.rs
+++ b/core/src/warc.rs
@@ -18,7 +18,7 @@ use crate::exponential_backoff::ExponentialBackoff;
use crate::{Error, Result, WarcSource};
use std::collections::BTreeMap;
use std::fs::File;
-use std::io::{BufRead, BufReader, Cursor, Read, Seek, SeekFrom, Write};
+use std::io::{BufRead, BufReader, Cursor, Read, Seek, Write};
use std::path::Path;
use std::time::Duration;
@@ -125,7 +125,7 @@ impl WarcFile {
let f = File::open(Path::new(folder).join(name))?;
let mut reader = BufReader::new(f);
- buf.seek(SeekFrom::Start(0))?;
+ buf.rewind()?;
std::io::copy(&mut reader, buf)?;
Ok(())
}
@@ -151,7 +151,7 @@ impl WarcFile {
let mut stream = res.bytes_stream();
- buf.seek(SeekFrom::Start(0))?;
+ buf.rewind()?;
while let Some(chunk) = stream.next().await {
std::io::copy(&mut &chunk?[..], buf)?;
}
diff --git a/core/src/webgraph/centrality/online_harmonic.rs b/core/src/webgraph/centrality/online_harmonic.rs
index 87acd6a2..433443fd 100644
--- a/core/src/webgraph/centrality/online_harmonic.rs
+++ b/core/src/webgraph/centrality/online_harmonic.rs
@@ -30,7 +30,6 @@ use std::path::Path;
use std::sync::{Arc, Mutex};
use itertools::Itertools;
-use rayon::prelude::*;
use serde::{Deserialize, Serialize};
use crate::intmap::IntMap;
diff --git a/core/src/webgraph/executor.rs b/core/src/webgraph/executor.rs
index a5041424..46226517 100644
--- a/core/src/webgraph/executor.rs
+++ b/core/src/webgraph/executor.rs
@@ -38,7 +38,7 @@ impl Executor {
pub fn with_threads(num_threads: usize, prefix: &'static str) -> Result<Executor> {
let pool = ThreadPoolBuilder::new()
.num_threads(num_threads)
- .thread_name(move |num| format!("{}{}", prefix, num))
+ .thread_name(move |num| format!("{prefix}{num}"))
.build()?;
Ok(Executor::ThreadPool(pool))
}
diff --git a/scripts/export_crossencoder b/scripts/export_crossencoder
index fba0becb..83316637 100755
--- a/scripts/export_crossencoder
+++ b/scripts/export_crossencoder
@@ -1,4 +1,13 @@
-#!/bin/bash
-pip3 install transformers[onnx] torch
-python3 -m transformers.onnx --model=cross-encoder/ms-marco-MiniLM-L-6-v2 data/
-mv data/model.onnx data/crossencoder.onnx
+#!/usr/bin/env python3
+import os
+import transformers
+
+MODEL = "cross-encoder/ms-marco-MiniLM-L-6-v2"
+
+os.system("pip3 install transformers[onnx] torch")
+os.system("mkdir data/cross_encoder")
+os.system(
+ f"python3 -m transformers.onnx --feature sequence-classification --model={MODEL} data/cross_encoder")
+
+tokenizer = transformers.AutoTokenizer.from_pretrained(MODEL)
+tokenizer.save_pretrained("data/cross_encoder")