Skip to content

web search done right

Stream output from summarization in iterator

Mikkel Denker0309386dfa79
diff --git a/Cargo.lock b/Cargo.lock
index 599b2c5d..8bf1417a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -48,9 +48,9 @@ dependencies = [
[[package]]
name = "anyhow"
-version = "1.0.68"
+version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61"
+checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800"
[[package]]
name = "arc-swap"
@@ -130,9 +130,9 @@ dependencies = [
[[package]]
name = "async-trait"
-version = "0.1.61"
+version = "0.1.64"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "705339e0e4a9690e2908d2b3d049d85682cf19fbd5782494498fbf7003a6a282"
+checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2"
dependencies = [
"proc-macro2",
"quote",
@@ -263,9 +263,9 @@ dependencies = [
[[package]]
name = "bindgen"
-version = "0.60.1"
+version = "0.64.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "062dddbc1ba4aca46de6338e2bf87771414c335f7b2f2036e8f3e9befebf88e6"
+checksum = "c4243e6031260db77ede97ad86c27e501d646a27ab57b59a574f725d98ab1fb4"
dependencies = [
"bitflags",
"cexpr",
@@ -278,6 +278,7 @@ dependencies = [
"regex",
"rustc-hash",
"shlex",
+ "syn",
]
[[package]]
@@ -360,15 +361,15 @@ dependencies = [
[[package]]
name = "bumpalo"
-version = "3.11.1"
+version = "3.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba"
+checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535"
[[package]]
name = "bytemuck"
-version = "1.12.3"
+version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aaa3a8d9a1ca92e282c96a32d6511b695d7d994d1d102ba85d279f9b2756947f"
+checksum = "c041d3eab048880cb0b86b256447da3f18859a163c3b8d8893f4e6368abe6393"
[[package]]
name = "byteorder"
@@ -378,9 +379,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
[[package]]
name = "bytes"
-version = "1.3.0"
+version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c"
+checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be"
[[package]]
name = "bzip2"
@@ -434,9 +435,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
[[package]]
name = "cc"
-version = "1.0.78"
+version = "1.0.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d"
+checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f"
dependencies = [
"jobserver",
]
@@ -578,7 +579,7 @@ dependencies = [
"lazy_static",
"libc",
"unicode-width",
- "windows-sys",
+ "windows-sys 0.42.0",
]
[[package]]
@@ -844,9 +845,9 @@ dependencies = [
[[package]]
name = "cxx"
-version = "1.0.86"
+version = "1.0.90"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "51d1075c37807dcf850c379432f0df05ba52cc30f279c5cfc43cc221ce7f8579"
+checksum = "90d59d9acd2a682b4e40605a242f6670eaa58c5957471cbf85e8aa6a0b97a5e8"
dependencies = [
"cc",
"cxxbridge-flags",
@@ -856,9 +857,9 @@ dependencies = [
[[package]]
name = "cxx-build"
-version = "1.0.86"
+version = "1.0.90"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5044281f61b27bc598f2f6647d480aed48d2bf52d6eb0b627d84c0361b17aa70"
+checksum = "ebfa40bda659dd5c864e65f4c9a2b0aff19bea56b017b9b77c73d3766a453a38"
dependencies = [
"cc",
"codespan-reporting",
@@ -871,15 +872,15 @@ dependencies = [
[[package]]
name = "cxxbridge-flags"
-version = "1.0.86"
+version = "1.0.90"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61b50bc93ba22c27b0d31128d2d130a0a6b3d267ae27ef7e4fae2167dfe8781c"
+checksum = "457ce6757c5c70dc6ecdbda6925b958aae7f959bda7d8fb9bde889e34a09dc03"
[[package]]
name = "cxxbridge-macro"
-version = "1.0.86"
+version = "1.0.90"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "39e61fda7e62115119469c7b3591fd913ecca96fb766cfd3f2e2502ab7bc87a5"
+checksum = "ebf883b7aacd7b2aeb2a7b338648ee19f57c140d4ee8e52c68979c6b2f7f2263"
dependencies = [
"proc-macro2",
"quote",
@@ -1048,9 +1049,9 @@ dependencies = [
[[package]]
name = "either"
-version = "1.8.0"
+version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797"
+checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91"
[[package]]
name = "ena"
@@ -1069,9 +1070,9 @@ checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
[[package]]
name = "encoding_rs"
-version = "0.8.31"
+version = "0.8.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b"
+checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394"
dependencies = [
"cfg-if",
]
@@ -1087,9 +1088,9 @@ dependencies = [
[[package]]
name = "exr"
-version = "1.5.2"
+version = "1.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8eb5f255b5980bb0c8cf676b675d1a99be40f316881444f44e0462eaf5df5ded"
+checksum = "e8af5ef47e2ed89d23d0ecbc1b681b30390069de70260937877514377fc24feb"
dependencies = [
"bit_field",
"flume",
@@ -1098,6 +1099,7 @@ dependencies = [
"miniz_oxide",
"smallvec",
"threadpool",
+ "zune-inflate",
]
[[package]]
@@ -1141,14 +1143,14 @@ dependencies = [
[[package]]
name = "filetime"
-version = "0.2.19"
+version = "0.2.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e884668cd0c7480504233e951174ddc3b382f7c2666e3b7310b5c4e7b0c37f9"
+checksum = "8a3de6e8d11b22ff9edc6d916f890800597d60f8b2da1caf2955c274638d6412"
dependencies = [
"cfg-if",
"libc",
"redox_syscall",
- "windows-sys",
+ "windows-sys 0.45.0",
]
[[package]]
@@ -1177,7 +1179,7 @@ dependencies = [
"futures-sink",
"nanorand",
"pin-project",
- "spin 0.9.4",
+ "spin 0.9.5",
]
[[package]]
@@ -1247,9 +1249,9 @@ dependencies = [
[[package]]
name = "futures"
-version = "0.3.25"
+version = "0.3.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38390104763dc37a5145a53c29c63c1290b5d316d6086ec32c293f6736051bb0"
+checksum = "13e2792b0ff0340399d58445b88fd9770e3489eff258a4cbc1523418f12abf84"
dependencies = [
"futures-channel",
"futures-core",
@@ -1262,9 +1264,9 @@ dependencies = [
[[package]]
name = "futures-channel"
-version = "0.3.25"
+version = "0.3.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed"
+checksum = "2e5317663a9089767a1ec00a487df42e0ca174b61b4483213ac24448e4664df5"
dependencies = [
"futures-core",
"futures-sink",
@@ -1272,15 +1274,15 @@ dependencies = [
[[package]]
name = "futures-core"
-version = "0.3.25"
+version = "0.3.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac"
+checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608"
[[package]]
name = "futures-executor"
-version = "0.3.25"
+version = "0.3.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7acc85df6714c176ab5edf386123fafe217be88c0840ec11f199441134a074e2"
+checksum = "e8de0a35a6ab97ec8869e32a2473f4b1324459e14c29275d14b10cb1fd19b50e"
dependencies = [
"futures-core",
"futures-task",
@@ -1289,15 +1291,15 @@ dependencies = [
[[package]]
name = "futures-io"
-version = "0.3.25"
+version = "0.3.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb"
+checksum = "bfb8371b6fb2aeb2d280374607aeabfc99d95c72edfe51692e42d3d7f0d08531"
[[package]]
name = "futures-macro"
-version = "0.3.25"
+version = "0.3.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bdfb8ce053d86b91919aad980c220b1fb8401a9394410e1c289ed7e66b61835d"
+checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70"
dependencies = [
"proc-macro2",
"quote",
@@ -1306,21 +1308,21 @@ dependencies = [
[[package]]
name = "futures-sink"
-version = "0.3.25"
+version = "0.3.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9"
+checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364"
[[package]]
name = "futures-task"
-version = "0.3.25"
+version = "0.3.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea"
+checksum = "dcf79a1bf610b10f42aea489289c5a2c478a786509693b80cd39c44ccd936366"
[[package]]
name = "futures-util"
-version = "0.3.25"
+version = "0.3.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6"
+checksum = "9c1d6de3acfef38d2be4b1f543f553131788603495be83da675e180c8d6b7bd1"
dependencies = [
"futures-channel",
"futures-core",
@@ -1451,9 +1453,9 @@ dependencies = [
[[package]]
name = "heck"
-version = "0.4.0"
+version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9"
+checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
[[package]]
name = "hermit-abi"
@@ -1566,9 +1568,9 @@ checksum = "02296996cb8796d7c6e3bc2d9211b7802812d36999a51bb754123ead7d37d026"
[[package]]
name = "hyper"
-version = "0.14.23"
+version = "0.14.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "034711faac9d2166cb1baf1a2fb0b60b1f277f8492fd72176c17f3515e1abd3c"
+checksum = "5e011372fa0b68db8350aa7a248930ecc7839bf46d8485577d69f117a75f164c"
dependencies = [
"bytes",
"futures-channel",
@@ -1787,9 +1789,9 @@ dependencies = [
[[package]]
name = "js-sys"
-version = "0.3.60"
+version = "0.3.61"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47"
+checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730"
dependencies = [
"wasm-bindgen",
]
@@ -1880,9 +1882,9 @@ dependencies = [
[[package]]
name = "librocksdb-sys"
-version = "0.8.0+7.4.4"
+version = "0.8.3+7.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "611804e4666a25136fcc5f8cf425ab4d26c7f74ea245ffe92ea23b85b6420b5d"
+checksum = "557b255ff04123fcc176162f56ed0c9cd42d8f357cf55b3fabeb60f7413741b3"
dependencies = [
"bindgen",
"bzip2-sys",
@@ -2037,9 +2039,9 @@ dependencies = [
[[package]]
name = "matches"
-version = "0.1.9"
+version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f"
+checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5"
[[package]]
name = "matchit"
@@ -2163,7 +2165,7 @@ dependencies = [
"libc",
"log",
"wasi 0.11.0+wasi-snapshot-preview1",
- "windows-sys",
+ "windows-sys 0.42.0",
]
[[package]]
@@ -2249,9 +2251,9 @@ dependencies = [
[[package]]
name = "num-complex"
-version = "0.4.2"
+version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ae39348c8bc5fbd7f40c727a9925f03517afd2ab27d46702108b6a7e5414c19"
+checksum = "02e0d21255c828d6f128a1e41534206671e8c3ea0c62f32291e808dc82cff17d"
dependencies = [
"num-traits",
]
@@ -2348,7 +2350,7 @@ dependencies = [
[[package]]
name = "onnxruntime"
version = "0.0.14"
-source = "git+https://github.com/Cuely/onnxruntime-rs#b7b48c9a6d4a553572caf8f1a595c3721a923a4d"
+source = "git+https://github.com/Cuely/onnxruntime-rs#b8d281e0e6f1e180700e8df437f6a07a36fec931"
dependencies = [
"lazy_static",
"ndarray",
@@ -2360,7 +2362,7 @@ dependencies = [
[[package]]
name = "onnxruntime-sys"
version = "0.0.14"
-source = "git+https://github.com/Cuely/onnxruntime-rs#b7b48c9a6d4a553572caf8f1a595c3721a923a4d"
+source = "git+https://github.com/Cuely/onnxruntime-rs#b8d281e0e6f1e180700e8df437f6a07a36fec931"
dependencies = [
"flate2",
"tar",
@@ -2470,15 +2472,15 @@ dependencies = [
[[package]]
name = "parking_lot_core"
-version = "0.9.6"
+version = "0.9.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba1ef8814b5c993410bb3adfad7a5ed269563e4a2f90c41f5d85be7fb47133bf"
+checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521"
dependencies = [
"cfg-if",
"libc",
"redox_syscall",
"smallvec",
- "windows-sys",
+ "windows-sys 0.45.0",
]
[[package]]
@@ -2507,9 +2509,9 @@ checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e"
[[package]]
name = "petgraph"
-version = "0.6.2"
+version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e6d5014253a1331579ce62aa67443b4a658c5e7dd03d4bc6d302b94474888143"
+checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4"
dependencies = [
"fixedbitset",
"indexmap",
@@ -2716,9 +2718,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068"
[[package]]
name = "proc-macro2"
-version = "1.0.49"
+version = "1.0.51"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5"
+checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6"
dependencies = [
"unicode-ident",
]
@@ -2857,9 +2859,9 @@ dependencies = [
[[package]]
name = "rayon-core"
-version = "1.10.1"
+version = "1.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cac410af5d00ab6884528b4ab69d1e8e146e8d471201800fa1b4524126de6ad3"
+checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b"
dependencies = [
"crossbeam-channel",
"crossbeam-deque",
@@ -2924,11 +2926,11 @@ dependencies = [
[[package]]
name = "reqwest"
-version = "0.11.13"
+version = "0.11.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68cc60575865c7831548863cc02356512e3f1dc2f3f82cb837d7fc4cc8f3c97c"
+checksum = "21eed90ec8570952d53b772ecf8f206aa1ec9a3d76b2521c56c42973f2d91ee9"
dependencies = [
- "base64 0.13.1",
+ "base64 0.21.0",
"bytes",
"encoding_rs",
"futures-core",
@@ -2956,6 +2958,7 @@ dependencies = [
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
+ "wasm-streams",
"web-sys",
"winreg",
]
@@ -3152,7 +3155,7 @@ version = "0.1.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3"
dependencies = [
- "windows-sys",
+ "windows-sys 0.42.0",
]
[[package]]
@@ -3191,9 +3194,9 @@ dependencies = [
[[package]]
name = "security-framework"
-version = "2.7.0"
+version = "2.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2bc1bb97804af6631813c55739f771071e0f2ed33ee20b68c86ec505d906356c"
+checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254"
dependencies = [
"bitflags",
"core-foundation",
@@ -3204,9 +3207,9 @@ dependencies = [
[[package]]
name = "security-framework-sys"
-version = "2.6.1"
+version = "2.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556"
+checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4"
dependencies = [
"core-foundation-sys",
"libc",
@@ -3270,9 +3273,9 @@ dependencies = [
[[package]]
name = "serde_json"
-version = "1.0.91"
+version = "1.0.93"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883"
+checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76"
dependencies = [
"itoa 1.0.5",
"ryu",
@@ -3349,6 +3352,12 @@ dependencies = [
"libc",
]
+[[package]]
+name = "simd-adler32"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "14a5df39617d7c8558154693a1bb8157a4aab8179209540cc0b10e5dc24e0b18"
+
[[package]]
name = "siphasher"
version = "0.3.10"
@@ -3388,9 +3397,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
[[package]]
name = "spin"
-version = "0.9.4"
+version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f6002a767bff9e83f8eeecf883ecb8011875a21ae8da43bffb817a57e78cc09"
+checksum = "7dccf47db1b41fa1573ed27ccf5e08e3ca771cb994f776668c5ebda893b248fc"
dependencies = [
"lock_api",
]
@@ -3476,9 +3485,9 @@ dependencies = [
[[package]]
name = "sync_wrapper"
-version = "0.1.1"
+version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8"
+checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
[[package]]
name = "tantivy"
@@ -3766,9 +3775,9 @@ dependencies = [
[[package]]
name = "tinyvec_macros"
-version = "0.1.0"
+version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
+checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokenizers"
@@ -3807,9 +3816,9 @@ dependencies = [
[[package]]
name = "tokio"
-version = "1.24.1"
+version = "1.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d9f76183f91ecfb55e1d7d5602bd1d979e38a3a522fe900241cf195624d67ae"
+checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af"
dependencies = [
"autocfg",
"bytes",
@@ -3822,7 +3831,7 @@ dependencies = [
"signal-hook-registry",
"socket2",
"tokio-macros",
- "windows-sys",
+ "windows-sys 0.42.0",
]
[[package]]
@@ -3838,9 +3847,9 @@ dependencies = [
[[package]]
name = "tokio-native-tls"
-version = "0.3.0"
+version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b"
+checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
dependencies = [
"native-tls",
"tokio",
@@ -3859,9 +3868,9 @@ dependencies = [
[[package]]
name = "tokio-util"
-version = "0.7.4"
+version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740"
+checksum = "bc6a3b08b64e6dfad376fa2432c7b1f01522e37a623c3050bc95db2d3ff21583"
dependencies = [
"bytes",
"futures-core",
@@ -3873,9 +3882,9 @@ dependencies = [
[[package]]
name = "toml"
-version = "0.5.10"
+version = "0.5.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1333c76748e868a4d9d1017b5ab53171dfd095f70c712fdb4653a406547f598f"
+checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234"
dependencies = [
"serde",
]
@@ -4020,9 +4029,9 @@ dependencies = [
[[package]]
name = "unicode-bidi"
-version = "0.3.8"
+version = "0.3.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992"
+checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58"
[[package]]
name = "unicode-ident"
@@ -4050,9 +4059,9 @@ dependencies = [
[[package]]
name = "unicode-segmentation"
-version = "1.10.0"
+version = "1.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fdbf052a0783de01e944a6ce7a8cb939e295b1e7be835a1112c3b9a7f047a5a"
+checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36"
[[package]]
name = "unicode-width"
@@ -4080,9 +4089,9 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
[[package]]
name = "ureq"
-version = "2.6.1"
+version = "2.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "733b5ad78377302af52c0dbcb2623d78fe50e4b3bf215948ff29e9ee031d8566"
+checksum = "338b31dd1314f68f3aabf3ed57ab922df95ffcd902476ca7ba3c4ce7b908c46d"
dependencies = [
"base64 0.13.1",
"flate2",
@@ -4131,9 +4140,9 @@ checksum = "5190c9442dcdaf0ddd50f37420417d219ae5261bbf5db120d0f9bab996c9cba1"
[[package]]
name = "uuid"
-version = "1.2.2"
+version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "422ee0de9031b5b948b97a8fc04e3aa35230001a722ddd27943e0be31564ce4c"
+checksum = "1674845326ee10d37ca60470760d4288a6f80f304007d92e5c53bab78c9cfd79"
dependencies = [
"getrandom 0.2.8",
"serde",
@@ -4204,9 +4213,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasm-bindgen"
-version = "0.2.83"
+version = "0.2.84"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268"
+checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b"
dependencies = [
"cfg-if",
"wasm-bindgen-macro",
@@ -4214,9 +4223,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-backend"
-version = "0.2.83"
+version = "0.2.84"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142"
+checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9"
dependencies = [
"bumpalo",
"log",
@@ -4229,9 +4238,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-futures"
-version = "0.4.33"
+version = "0.4.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d"
+checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454"
dependencies = [
"cfg-if",
"js-sys",
@@ -4241,9 +4250,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
-version = "0.2.83"
+version = "0.2.84"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810"
+checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@@ -4251,9 +4260,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
-version = "0.2.83"
+version = "0.2.84"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c"
+checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6"
dependencies = [
"proc-macro2",
"quote",
@@ -4264,15 +4273,28 @@ dependencies = [
[[package]]
name = "wasm-bindgen-shared"
-version = "0.2.83"
+version = "0.2.84"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d"
+
+[[package]]
+name = "wasm-streams"
+version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f"
+checksum = "6bbae3363c08332cadccd13b67db371814cd214c2524020932f0804b8cf7c078"
+dependencies = [
+ "futures-util",
+ "js-sys",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+]
[[package]]
name = "web-sys"
-version = "0.3.60"
+version = "0.3.61"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f"
+checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97"
dependencies = [
"js-sys",
"wasm-bindgen",
@@ -4372,6 +4394,30 @@ dependencies = [
"windows_x86_64_msvc 0.42.1",
]
+[[package]]
+name = "windows-sys"
+version = "0.45.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
+dependencies = [
+ "windows-targets",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.42.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7"
+dependencies = [
+ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc 0.42.1",
+ "windows_i686_gnu 0.42.1",
+ "windows_i686_msvc 0.42.1",
+ "windows_x86_64_gnu 0.42.1",
+ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc 0.42.1",
+]
+
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.42.1"
@@ -4508,11 +4554,20 @@ dependencies = [
[[package]]
name = "zstd-sys"
-version = "2.0.5+zstd.1.5.2"
+version = "2.0.7+zstd.1.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "edc50ffce891ad571e9f9afe5039c4837bede781ac4bb13052ed7ae695518596"
+checksum = "94509c3ba2fe55294d752b79842c530ccfab760192521df74a081a78d2b3c7f5"
dependencies = [
"cc",
"libc",
"pkg-config",
]
+
+[[package]]
+name = "zune-inflate"
+version = "0.2.42"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c473377c11c4a3ac6a2758f944cd336678e9c977aa0abf54f6450cf77e902d6d"
+dependencies = [
+ "simd-adler32",
+]
diff --git a/core/src/entrypoint/entity.rs b/core/src/entrypoint/entity.rs
index af3334bd..e584404d 100644
--- a/core/src/entrypoint/entity.rs
+++ b/core/src/entrypoint/entity.rs
@@ -113,7 +113,6 @@ impl EntityIndexer {
let reader = BufReader::new(MultiBzDecoder::new(reader));
let mut index = EntityIndex::open(output_path)?;
EntityIterator::from(reader)
- .into_iter()
.filter(|entity| !entity.categories.is_empty())
.take(200_000)
.for_each(|entity| {
diff --git a/core/src/query/parser.rs b/core/src/query/parser.rs
index ce6beddf..26410db1 100644
--- a/core/src/query/parser.rs
+++ b/core/src/query/parser.rs
@@ -207,7 +207,6 @@ impl Term {
fields
.iter()
.filter(|(field, _)| ALL_FIELDS[field.field_id() as usize].is_searchable())
- .into_iter()
.map(|(field, entry)| {
(
Occur::Should,
@@ -232,7 +231,6 @@ impl Term {
| Field::Text(TextField::SiteNoTokenizer)
)
})
- .into_iter()
.map(|(field, entry)| {
(
Occur::Should,
diff --git a/core/src/spell/dictionary.rs b/core/src/spell/dictionary.rs
index 5cbf8cce..6acd5551 100644
--- a/core/src/spell/dictionary.rs
+++ b/core/src/spell/dictionary.rs
@@ -268,7 +268,6 @@ impl<const TOP_N: usize> Dictionary<TOP_N> {
self.cache
.entry(
term.chars()
- .into_iter()
.map(|c| c.to_ascii_lowercase())
.filter(|c| !matches!(c, ',' | '.' | '\\' | '=' | '*' | '(' | ')'))
.collect(),
diff --git a/core/src/summarizer.rs b/core/src/summarizer.rs
index f857f164..37ced79d 100644
--- a/core/src/summarizer.rs
+++ b/core/src/summarizer.rs
@@ -314,9 +314,70 @@ impl AbstractiveModel {
}
pub fn summarize(&self, text: &str, config: GenerationConfig) -> Result<String> {
- let encoded_text = self.tokenizer.encode(text, true)?;
+ let generator = BeamGenerator {
+ model: self,
+ config,
+ state: Some(BeamState::Encoding {
+ text: text.to_string(),
+ }),
+ };
+
+ Ok(generator.collect())
+ }
+}
+
+struct BeamGenerator<'a> {
+ model: &'a AbstractiveModel,
+ config: GenerationConfig,
+ state: Option<BeamState>,
+}
+
+impl<'a> Iterator for BeamGenerator<'a> {
+ type Item = String;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ self.state
+ .take()
+ .and_then(|state| match state.step(self.model, &self.config) {
+ Ok((state, res)) => {
+ self.state = Some(state);
+ res
+ }
+ Err(err) => {
+ tracing::error!("Encountered an error while generating summary: {err}");
+ None
+ }
+ })
+ }
+}
+
+enum BeamState {
+ Encoding {
+ text: String,
+ },
+ Decoding {
+ encoder_hidden_states: ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>,
+ attention_mask: ArrayBase<OwnedRepr<i64>, Dim<IxDynImpl>>,
+ },
+ DecodingWithPast {
+ encoder_hidden_states: ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>,
+ attention_mask: ArrayBase<OwnedRepr<i64>, Dim<IxDynImpl>>,
+ caches: Vec<ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>>,
+ beams: Vec<Beam>,
+ generated_tokens: u32,
+ /// all beams agree on the tokens up to this index
+ beam_token_agreement_idx: usize,
+ },
+ Remaining {
+ terms: VecDeque<String>,
+ },
+}
+
+impl BeamState {
+ fn encode_step(model: &AbstractiveModel, text: String) -> Result<Self> {
+ let encoded_text = model.tokenizer.encode(text.as_str(), true)?;
- let mut ids = encoded_text
+ let ids = encoded_text
.get_ids()
.iter()
.map(|i| *i as i64)
@@ -332,292 +393,492 @@ impl AbstractiveModel {
let encoder_num_tokens = ids.len();
- let mut decoder = self
- .decoder
- .lock()
- .expect("Failed to get lock. Maybe another thread crashed?");
+ let encoder_hidden_states = model.encode(ids, attention_mask.clone())?;
- let mut decoder_with_past = self
- .decoder_with_past
+ let attention_mask: ArrayBase<OwnedRepr<i64>, Dim<IxDynImpl>> =
+ ArrayBase::from_vec(attention_mask)
+ .into_shape((1, encoder_num_tokens))
+ .unwrap()
+ .into_dyn();
+
+ Ok(BeamState::Decoding {
+ encoder_hidden_states,
+ attention_mask,
+ })
+ }
+
+ fn run_decoder_model(
+ model: &AbstractiveModel,
+ ids: &[i64],
+ encoder_hidden_states: ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>,
+ attention_mask: ArrayBase<OwnedRepr<i64>, Dim<IxDynImpl>>,
+ ) -> Result<Vec<ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>>> {
+ let mut decoder = model
+ .decoder
.lock()
.expect("Failed to get lock. Maybe another thread crashed?");
- let arr_encoder_hidden_states = self.encode(ids.clone(), attention_mask.clone())?;
-
- let arr_attention_mask = ArrayBase::from_vec(attention_mask.clone())
- .into_shape((1, encoder_num_tokens))
+ let num_tokens = ids.len();
+ let arr_ids = ArrayBase::from_vec(ids.to_vec())
+ .into_shape((1, num_tokens))
.unwrap()
.into_dyn();
- let mut beams = Vec::new();
-
- ids = vec![self.begin_decoder_token as i64, self.bos_token_id as i64];
- let mut generated_tokens = 0;
+ let decoder_input = vec![
+ TypedArray::I64(attention_mask),
+ TypedArray::I64(arr_ids),
+ TypedArray::F32(encoder_hidden_states),
+ ];
- let mut caches = Vec::new();
+ let output = decoder.run(decoder_input)?;
- loop {
- generated_tokens += 1;
+ Ok(output.into_iter().map(|t| t.clone().into_owned()).collect())
+ }
- if beams.is_empty() {
- let num_tokens = ids.len();
- let arr_ids = ArrayBase::from_vec(ids.clone())
- .into_shape((1, num_tokens))
- .unwrap()
- .into_dyn();
+ fn create_beams(
+ decoder_output: &[ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>],
+ ids: Vec<i64>,
+ num_beams: u32,
+ bos_token: usize,
+ ) -> Vec<Beam> {
+ let mut next_scores = BinaryHeap::with_capacity(num_beams as usize + 1);
+
+ let all_logits = &decoder_output[0];
+ let vocab_size = all_logits.shape()[2];
+ let mut scores = Vec::with_capacity(vocab_size);
+
+ for token_id in 0..vocab_size {
+ let score: f32 = all_logits[[
+ all_logits.shape()[0] - 1,
+ all_logits.shape()[1] - 1,
+ token_id,
+ ]];
+ scores.push(score);
+ }
- let decoder_input = vec![
- TypedArray::I64(arr_attention_mask.clone()),
- TypedArray::I64(arr_ids),
- TypedArray::F32(arr_encoder_hidden_states.clone()),
- ];
+ softmax(&mut scores);
- let decoder_output = decoder.run(decoder_input)?;
- let mut next_scores = BinaryHeap::with_capacity(config.num_beams as usize + 1);
+ for (token_id, score) in scores.into_iter().enumerate() {
+ if token_id == bos_token {
+ continue;
+ }
- let all_logits = &decoder_output[0];
- let vocab_size = all_logits.shape()[2];
- let mut scores = Vec::with_capacity(vocab_size);
+ let scored_token = Reverse(ScoredToken {
+ token: token_id as i64,
+ score: score.log2(),
+ });
- for token_id in 0..vocab_size {
- let score: f32 = all_logits[[0, num_tokens - 1, token_id]];
- scores.push(score);
+ if next_scores.len() < num_beams as usize {
+ next_scores.push(scored_token);
+ } else if let Some(mut worst) = next_scores.peek_mut() {
+ if worst.0.score < scored_token.0.score {
+ *worst = scored_token
}
+ }
+ }
- softmax(&mut scores);
+ let mut beams = Vec::new();
+ for scored_token in next_scores.into_iter().map(|t| t.0) {
+ let mut ids = ids.clone();
+ ids.push(scored_token.token);
+
+ beams.push(Beam {
+ input_ids: ids,
+ score: scored_token.score,
+ })
+ }
- for (token_id, score) in scores.into_iter().enumerate() {
- if token_id == self.bos_token_id {
- continue;
- }
+ beams
+ }
- let scored_token = Reverse(ScoredToken {
- token: token_id as i64,
- score: score.log2(),
- });
-
- if next_scores.len() < config.num_beams as usize {
- next_scores.push(scored_token);
- } else if let Some(mut worst) = next_scores.peek_mut() {
- if worst.0.score < scored_token.0.score {
- *worst = scored_token
- }
- }
+ fn create_caches(
+ decoder_output: Vec<ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>>,
+ num_beams: u32,
+ ) -> Vec<ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>> {
+ let num_outputs = decoder_output.len();
+
+ decoder_output
+ .into_iter()
+ .map(|t| t.into_owned())
+ .map(|mut t| {
+ let orig = t.clone();
+
+ for _ in 0..num_beams - 1 {
+ t.append(Axis(0), orig.view()).unwrap();
}
- let num_outputs = decoder_output.len();
+ t
+ })
+ .skip(1)
+ .take(num_outputs - 2) // skip 'encoder_hidden_states' since this is not changed by the decoder model
+ .collect::<Vec<_>>()
+ }
- caches = decoder_output
- .into_iter()
- .map(|t| t.clone().into_owned())
- .map(|mut t| {
- let orig = t.clone();
+ fn run_decoder_model_with_past(
+ model: &AbstractiveModel,
+ beams: &[Beam],
+ encoder_hidden_states: &ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>,
+ attention_mask: &ArrayBase<OwnedRepr<i64>, Dim<IxDynImpl>>,
+ mut caches: Vec<ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>>,
+ ) -> Result<Vec<ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>>> {
+ let mut decoder_with_past = model
+ .decoder_with_past
+ .lock()
+ .expect("Failed to get lock. Maybe another thread crashed?");
- for _ in 0..config.num_beams - 1 {
- t.append(Axis(0), orig.view()).unwrap();
- }
+ let mut combined_attention_mask = Vec::new();
+ let mut combined_encoder_hidden_states = Vec::with_capacity(beams.len());
+ let mut combined_ids = Vec::new();
- t
- })
- .skip(1)
- .take(num_outputs - 2) // skip 'encoder_hidden_states' since this is not changed by the decoder model
- .collect::<Vec<_>>();
+ // TODO: Only advance non-finished beams
- for scored_token in next_scores.into_iter().map(|t| t.0) {
- let mut ids = ids.clone();
- ids.push(scored_token.token);
+ for beam in beams {
+ combined_ids.push(*beam.input_ids.last().unwrap());
- beams.push(Beam {
- input_ids: ids,
- score: scored_token.score,
- })
- }
- } else {
- let mut combined_attention_mask = Vec::new();
- let mut combined_encoder_hidden_states = Vec::with_capacity(beams.len());
- let mut combined_ids = Vec::new();
+ for mask in attention_mask {
+ combined_attention_mask.push(*mask);
+ }
- // TODO: Only advance non-finished beams
+ for state in encoder_hidden_states {
+ combined_encoder_hidden_states.push(*state);
+ }
+ }
- for beam in &beams {
- combined_ids.push(*beam.input_ids.last().unwrap());
+ let combined_ids = ArrayBase::<OwnedRepr<_>, _>::from_vec(combined_ids)
+ .into_shape((beams.len(), 1))
+ .unwrap()
+ .into_dyn();
- for mask in &attention_mask {
- combined_attention_mask.push(*mask);
- }
+ let mut attention_shape = attention_mask.shape().to_vec();
+ attention_shape[0] = beams.len();
- for state in &arr_encoder_hidden_states {
- combined_encoder_hidden_states.push(*state);
- }
- }
+ let combined_attention_mask =
+ ArrayBase::<OwnedRepr<_>, _>::from_vec(combined_attention_mask)
+ .into_shape(attention_shape.as_slice())
+ .unwrap();
- let combined_ids = ArrayBase::<OwnedRepr<_>, _>::from_vec(combined_ids)
- .into_shape((beams.len(), 1))
- .unwrap()
- .into_dyn();
+ let mut state_shape = encoder_hidden_states.shape().to_vec();
+ state_shape[0] = beams.len();
- let mut attention_shape = arr_attention_mask.shape().to_vec();
- attention_shape[0] = beams.len();
+ let combined_encoder_hidden_states =
+ ArrayBase::<OwnedRepr<_>, _>::from_vec(combined_encoder_hidden_states)
+ .into_shape(state_shape.as_slice())
+ .unwrap();
- let combined_attention_mask =
- ArrayBase::<OwnedRepr<_>, _>::from_vec(combined_attention_mask)
- .into_shape(attention_shape.as_slice())
- .unwrap();
+ let mut decoder_input = vec![
+ TypedArray::I64(combined_attention_mask),
+ TypedArray::I64(combined_ids),
+ TypedArray::F32(combined_encoder_hidden_states),
+ ];
- let mut state_shape = arr_encoder_hidden_states.shape().to_vec();
- state_shape[0] = beams.len();
+ for tensor in caches.drain(0..) {
+ decoder_input.push(TypedArray::F32(tensor));
+ }
- let combined_encoder_hidden_states =
- ArrayBase::<OwnedRepr<_>, _>::from_vec(combined_encoder_hidden_states)
- .into_shape(state_shape.as_slice())
- .unwrap();
+ let res = decoder_with_past.run(decoder_input)?;
- let mut decoder_input = vec![
- TypedArray::I64(combined_attention_mask),
- TypedArray::I64(combined_ids),
- TypedArray::F32(combined_encoder_hidden_states),
- ];
+ Ok(res.into_iter().map(|t| t.clone().into_owned()).collect())
+ }
- for tensor in caches.drain(0..) {
- decoder_input.push(TypedArray::F32(tensor));
- }
+ fn new_possible_beams(
+ decoder_output: &[ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>],
+ beams: &[Beam],
+ force_min_tokens: Option<u32>,
+ generated_tokens: u32,
+ eos_token_id: usize,
+ length_penalty: f32,
+ ) -> Vec<PossibleBeam> {
+ let all_logits = &decoder_output[0];
+ let vocab_size = all_logits.shape()[2];
+
+ let mut beam_tok_scores = Vec::with_capacity(beams.len());
+ for beam_idx in 0..beams.len() {
+ let mut scores = Vec::with_capacity(vocab_size);
+
+ for token_id in 0..vocab_size {
+ scores.push(all_logits[[beam_idx, 0, token_id]]);
+ }
- let decoder_output: Vec<OrtOwnedTensor<f32, Dim<IxDynImpl>>> =
- decoder_with_past.run(decoder_input)?;
+ softmax(&mut scores);
+ beam_tok_scores.push(scores)
+ }
- let all_logits = &decoder_output[0];
- let vocab_size = all_logits.shape()[2];
+ let mut new_beams = BinaryHeap::with_capacity(beams.len());
- let mut beam_tok_scores = Vec::with_capacity(beams.len());
- for beam_idx in 0..beams.len() {
- let mut scores = Vec::with_capacity(vocab_size);
+ let mut skip_eos = false;
- for token_id in 0..vocab_size {
- scores.push(all_logits[[beam_idx, 0, token_id]]);
- }
+ if let Some(min_tokens) = force_min_tokens {
+ if generated_tokens < min_tokens {
+ skip_eos = true;
+ }
+ }
- softmax(&mut scores);
- beam_tok_scores.push(scores)
+ for beam_idx in 0..beams.len() {
+ for new_token in 0..vocab_size {
+ if skip_eos && new_token == eos_token_id {
+ continue;
}
- let num_outputs = decoder_output.len();
+ let tok_score = beam_tok_scores[beam_idx][new_token];
- // skip(num_outputs-2) to skip 'encoder_hidden_states' since this is not changed by the decoder model
- let output_caches: Vec<_> = decoder_output
- .iter()
- .skip(1)
- .take(num_outputs - 2)
- .collect();
+ let tok_score = tok_score.log2();
- let mut new_beams = BinaryHeap::with_capacity(beams.len());
+ let new_score = beams[beam_idx].score + tok_score;
- for beam_idx in 0..beams.len() {
- for new_token in 0..vocab_size {
- let tok_score = beam_tok_scores[beam_idx][new_token];
+ let possible_beam = PossibleBeam {
+ beam_idx,
+ beam_length: beams[beam_idx].len(),
+ new_token: new_token as i64,
+ length_penalty,
+ eos_token: eos_token_id as i64,
+ new_score,
+ };
- let tok_score = tok_score.log2();
+ if new_beams.len() < beams.len() {
+ new_beams.push(Reverse(possible_beam));
+ } else if let Some(mut worst) = new_beams.peek_mut() {
+ if possible_beam > worst.0 {
+ *worst = Reverse(possible_beam);
+ }
+ }
+ }
+ }
- let new_score = beams[beam_idx].score + tok_score;
+ new_beams.into_iter().map(|r| r.0).collect()
+ }
- let possible_beam = PossibleBeam {
- beam_idx,
- beam_length: beams[beam_idx].len(),
- new_token: new_token as i64,
- length_penalty: config.length_penalty,
- eos_token: self.eos_token_id as i64,
- new_score,
- };
+ fn new_caches(
+ decoder_output: &[ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>],
+ new_beams: &[PossibleBeam],
+ ) -> Vec<ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>> {
+ let num_outputs = decoder_output.len();
+ // skip(num_outputs-2) to skip 'encoder_hidden_states' since this is not changed by the decoder model
+ let output_caches: Vec<_> = decoder_output
+ .iter()
+ .skip(1)
+ .take(num_outputs - 2)
+ .collect();
+
+ let mut new_caches = Vec::with_capacity(output_caches.len());
+
+ for _ in &output_caches {
+ new_caches.push(Vec::with_capacity(100));
+ }
- if new_beams.len() < beams.len() {
- new_beams.push(Reverse(possible_beam));
- } else if let Some(mut worst) = new_beams.peek_mut() {
- if possible_beam > worst.0 {
- *worst = Reverse(possible_beam);
- }
- }
+ for possible_beam in new_beams {
+ for (new_cache, old_cache) in new_caches.iter_mut().zip_eq(output_caches.iter()) {
+ let shape = old_cache.shape();
+
+ new_cache.push(
+ old_cache
+ .index_axis(Axis(0), possible_beam.beam_idx)
+ .into_shape((1, shape[1], shape[2], shape[3]))
+ .unwrap()
+ .into_dyn(),
+ );
+ }
+ }
+
+ new_caches
+ .into_iter()
+ .map(|cache| onnxruntime::ndarray::concatenate(Axis(0), cache.as_slice()).unwrap())
+ .collect()
+ }
+
+ fn cement_beams(
+ beams: Vec<Beam>,
+ possible_beams: Vec<PossibleBeam>,
+ eos_token_id: i64,
+ ) -> Vec<Beam> {
+ possible_beams
+ .into_iter()
+ .map(|possible_beam| {
+ if beams[possible_beam.beam_idx].is_finished(eos_token_id) {
+ beams[possible_beam.beam_idx].clone()
+ } else {
+ let mut input_ids = beams[possible_beam.beam_idx].input_ids.clone();
+ input_ids.push(possible_beam.new_token);
+ Beam {
+ input_ids,
+ score: possible_beam.new_score,
}
}
+ })
+ .collect()
+ }
+
+ fn output_from_best(
+ model: &AbstractiveModel,
+ new_beams: Vec<Beam>,
+ beam_agreement_index: usize,
+ ) -> BeamState {
+ let best_beam = new_beams
+ .into_iter()
+ .max_by(|a, b| a.score.total_cmp(&b.score))
+ .unwrap();
+
+ let terms: VecDeque<_> = best_beam
+ .input_ids
+ .into_iter()
+ .skip(beam_agreement_index)
+ .map(|i| i as u32)
+ .map(|tok| model.tokenizer.decode(vec![tok], true).unwrap())
+ .collect();
+
+ BeamState::Remaining { terms }
+ }
- let mut new_caches = Vec::with_capacity(output_caches.len());
+ fn check_early_stopping(early_stopping: Option<&EarlyStopping>, generated_tokens: u32) -> bool {
+ early_stopping
+ .map(|early_stopping| match early_stopping {
+ EarlyStopping::MaxTokens { max_new_tokens } => generated_tokens >= *max_new_tokens,
+ })
+ .unwrap_or(false)
+ }
- for _ in &output_caches {
- new_caches.push(Vec::with_capacity(100));
+ fn all_beams_agree(beams: &[Beam], beam_token_agreement_idx: usize) -> bool {
+ if beams
+ .iter()
+ .any(|beam| beam_token_agreement_idx >= beam.len())
+ {
+ return false;
+ }
+
+ beams
+ .iter()
+ .map(|beam| beam.input_ids[beam_token_agreement_idx])
+ .all_equal()
+ }
+
+ fn step<'a>(
+ mut self,
+ model: &'a AbstractiveModel,
+ config: &'a GenerationConfig,
+ ) -> Result<(Self, Option<<BeamGenerator<'a> as Iterator>::Item>)> {
+ loop {
+ match self {
+ BeamState::Encoding { text } => {
+ self = Self::encode_step(model, text)?;
}
+ BeamState::Decoding {
+ encoder_hidden_states,
+ attention_mask,
+ } => {
+ let ids = vec![model.begin_decoder_token as i64, model.bos_token_id as i64];
+ let decoder_output = Self::run_decoder_model(
+ model,
+ &ids,
+ encoder_hidden_states.clone(),
+ attention_mask.clone(),
+ )?;
+
+ let beams = Self::create_beams(
+ &decoder_output,
+ ids,
+ config.num_beams,
+ model.bos_token_id,
+ );
+
+ let caches = Self::create_caches(decoder_output, config.num_beams);
+
+ self = BeamState::DecodingWithPast {
+ encoder_hidden_states,
+ attention_mask,
+ caches,
+ beams,
+ generated_tokens: 1,
+ beam_token_agreement_idx: 0,
+ };
+ }
+
+ BeamState::DecodingWithPast {
+ encoder_hidden_states,
+ attention_mask,
+ caches,
+ mut beams,
+ generated_tokens,
+ beam_token_agreement_idx,
+ } => {
+ let shortest_beam = beams
+ .iter()
+ .min_by(|a, b| a.input_ids.len().cmp(&b.input_ids.len()))
+ .unwrap();
- for possible_beam in &new_beams {
- for (new_cache, old_cache) in new_caches.iter_mut().zip_eq(output_caches.iter())
+ if (shortest_beam.input_ids.len() - beam_token_agreement_idx) as u32
+ > config.max_beam_divergence_tokens
{
- let shape = old_cache.shape();
-
- new_cache.push(
- old_cache
- .index_axis(Axis(0), possible_beam.0.beam_idx)
- .into_shape((1, shape[1], shape[2], shape[3]))
- .unwrap()
- .into_dyn(),
- );
+ let longest_beam = beams
+ .iter()
+ .max_by(|a, b| a.input_ids.len().cmp(&b.input_ids.len()))
+ .unwrap()
+ .clone();
+
+ let num_beams = beams.len();
+ beams = vec![longest_beam; num_beams];
}
- }
- let new_caches: Vec<_> = new_caches
- .into_iter()
- .map(|cache| {
- onnxruntime::ndarray::concatenate(Axis(0), cache.as_slice()).unwrap()
- })
- .collect();
-
- let new_beams: Vec<_> = new_beams
- .into_iter()
- .map(|r| r.0)
- .map(|possible_beam| {
- if beams[possible_beam.beam_idx].is_finished(self.eos_token_id as i64) {
- beams[possible_beam.beam_idx].clone()
- } else {
- let mut input_ids = beams[possible_beam.beam_idx].input_ids.clone();
- input_ids.push(possible_beam.new_token);
- Beam {
- input_ids,
- score: possible_beam.new_score,
- }
- }
- })
- .collect();
-
- beams = new_beams;
- caches = new_caches;
-
- if beams
- .iter()
- .all(|beam| beam.is_finished(self.eos_token_id as i64))
- {
- break;
- }
- }
+ if Self::all_beams_agree(&beams, beam_token_agreement_idx) {
+ let tok = beams[0].input_ids[beam_token_agreement_idx];
+ let decoded = model.tokenizer.decode(vec![tok as u32], true)?;
+
+ self = BeamState::DecodingWithPast {
+ encoder_hidden_states,
+ attention_mask,
+ caches,
+ beams,
+ generated_tokens,
+ beam_token_agreement_idx: beam_token_agreement_idx + 1,
+ };
- if let Some(min_tokens) = config.force_min_tokens {
- if generated_tokens < min_tokens {
- continue;
- }
- }
+ return Ok((self, Some(decoded)));
+ }
- if let Some(early_stopping) = &config.early_stopping {
- match early_stopping {
- EarlyStopping::MaxTokens { max_new_tokens } => {
- if generated_tokens >= *max_new_tokens {
- break;
- }
+ let decoder_output = Self::run_decoder_model_with_past(
+ model,
+ &beams,
+ &encoder_hidden_states,
+ &attention_mask,
+ caches,
+ )?;
+
+ let possible_beams = Self::new_possible_beams(
+ &decoder_output,
+ &beams,
+ config.force_min_tokens,
+ generated_tokens,
+ model.eos_token_id,
+ config.length_penalty,
+ );
+
+ let new_caches = Self::new_caches(&decoder_output, &possible_beams);
+
+ let new_beams =
+ Self::cement_beams(beams, possible_beams, model.eos_token_id as i64);
+
+ if Self::check_early_stopping(config.early_stopping.as_ref(), generated_tokens)
+ || new_beams
+ .iter()
+ .all(|beam| beam.is_finished(model.eos_token_id as i64))
+ {
+ self = Self::output_from_best(model, new_beams, beam_token_agreement_idx);
+ } else {
+ self = BeamState::DecodingWithPast {
+ encoder_hidden_states,
+ attention_mask,
+ caches: new_caches,
+ beams: new_beams,
+ generated_tokens: generated_tokens + 1,
+ beam_token_agreement_idx,
+ };
}
}
+ BeamState::Remaining { mut terms } => {
+ let term = terms.pop_front();
+
+ return Ok((BeamState::Remaining { terms }, term));
+ }
}
}
-
- let best_beam = beams
- .into_iter()
- .max_by(|a, b| a.score.total_cmp(&b.score))
- .unwrap();
-
- let res: Vec<_> = best_beam.input_ids.into_iter().map(|i| i as u32).collect();
- Ok(self.tokenizer.decode(res, true).unwrap())
}
}
@@ -716,8 +977,13 @@ impl Eq for PossibleBeam {}
pub struct GenerationConfig {
pub early_stopping: Option<EarlyStopping>,
pub force_min_tokens: Option<u32>,
+ /// Exponential penalty to the length that is used with beam-based generation.
+ /// It is applied as an exponent to the sequence length, which in turn is used to divide the score of the sequence.
+ /// Since the score is the log likelihood of the sequence (i.e. negative),
+ /// length_penalty > 0.0 promotes longer sequences, while length_penalty < 0.0 encourages shorter sequences
pub length_penalty: f32,
pub num_beams: u32,
+ pub max_beam_divergence_tokens: u32,
}
impl Default for GenerationConfig {
@@ -729,6 +995,7 @@ impl Default for GenerationConfig {
force_min_tokens: None,
length_penalty: 1.0,
num_beams: 5,
+ max_beam_divergence_tokens: 20,
}
}
}
@@ -788,7 +1055,7 @@ mod tests {
let config = GenerationConfig {
num_beams: 2,
length_penalty: 1.0,
- early_stopping: Some(EarlyStopping::MaxTokens { max_new_tokens: 32 }),
+ early_stopping: Some(EarlyStopping::MaxTokens { max_new_tokens: 16 }),
..Default::default()
};
@@ -796,6 +1063,7 @@ mod tests {
let summary = model.summarize(text, config).unwrap();
println!("Elapsed: {:?}", start.elapsed());
println!("{:?}", &summary);
+
assert!(summary.len() > 50);
assert!(summary.contains("Aristotle"));
}
diff --git a/core/src/webpage/region.rs b/core/src/webpage/region.rs
index edae215d..6852578e 100644
--- a/core/src/webpage/region.rs
+++ b/core/src/webpage/region.rs
@@ -22,8 +22,9 @@ use crate::{Error, Result};
use super::Webpage;
-#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, Serialize, Deserialize)]
+#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, Serialize, Deserialize, Default)]
pub enum Region {
+ #[default]
All,
Denmark,
France,
@@ -32,12 +33,6 @@ pub enum Region {
US,
}
-impl Default for Region {
- fn default() -> Self {
- Region::All
- }
-}
-
pub const ALL_REGIONS: [Region; 6] = [
Region::All,
Region::Denmark,