web search done right
Stream output from summarization in iterator
Mikkel Denker0309386dfa79
diff --git a/Cargo.lock b/Cargo.lockindex 599b2c5d..8bf1417a 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -48,9 +48,9 @@ dependencies = [[[package]]name = "anyhow"-version = "1.0.68"+version = "1.0.69"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61"+checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800"[[package]]name = "arc-swap"@@ -130,9 +130,9 @@ dependencies = [[[package]]name = "async-trait"-version = "0.1.61"+version = "0.1.64"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "705339e0e4a9690e2908d2b3d049d85682cf19fbd5782494498fbf7003a6a282"+checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2"dependencies = ["proc-macro2","quote",@@ -263,9 +263,9 @@ dependencies = [[[package]]name = "bindgen"-version = "0.60.1"+version = "0.64.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "062dddbc1ba4aca46de6338e2bf87771414c335f7b2f2036e8f3e9befebf88e6"+checksum = "c4243e6031260db77ede97ad86c27e501d646a27ab57b59a574f725d98ab1fb4"dependencies = ["bitflags","cexpr",@@ -278,6 +278,7 @@ dependencies = ["regex","rustc-hash","shlex",+ "syn",][[package]]@@ -360,15 +361,15 @@ dependencies = [[[package]]name = "bumpalo"-version = "3.11.1"+version = "3.12.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba"+checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535"[[package]]name = "bytemuck"-version = "1.12.3"+version = "1.13.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "aaa3a8d9a1ca92e282c96a32d6511b695d7d994d1d102ba85d279f9b2756947f"+checksum = "c041d3eab048880cb0b86b256447da3f18859a163c3b8d8893f4e6368abe6393"[[package]]name = "byteorder"@@ -378,9 +379,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"[[package]]name = "bytes"-version = "1.3.0"+version = "1.4.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c"+checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be"[[package]]name = "bzip2"@@ -434,9 +435,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"[[package]]name = "cc"-version = "1.0.78"+version = "1.0.79"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d"+checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f"dependencies = ["jobserver",]@@ -578,7 +579,7 @@ dependencies = ["lazy_static","libc","unicode-width",- "windows-sys",+ "windows-sys 0.42.0",][[package]]@@ -844,9 +845,9 @@ dependencies = [[[package]]name = "cxx"-version = "1.0.86"+version = "1.0.90"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "51d1075c37807dcf850c379432f0df05ba52cc30f279c5cfc43cc221ce7f8579"+checksum = "90d59d9acd2a682b4e40605a242f6670eaa58c5957471cbf85e8aa6a0b97a5e8"dependencies = ["cc","cxxbridge-flags",@@ -856,9 +857,9 @@ dependencies = [[[package]]name = "cxx-build"-version = "1.0.86"+version = "1.0.90"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5044281f61b27bc598f2f6647d480aed48d2bf52d6eb0b627d84c0361b17aa70"+checksum = "ebfa40bda659dd5c864e65f4c9a2b0aff19bea56b017b9b77c73d3766a453a38"dependencies = ["cc","codespan-reporting",@@ -871,15 +872,15 @@ dependencies = [[[package]]name = "cxxbridge-flags"-version = "1.0.86"+version = "1.0.90"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "61b50bc93ba22c27b0d31128d2d130a0a6b3d267ae27ef7e4fae2167dfe8781c"+checksum = "457ce6757c5c70dc6ecdbda6925b958aae7f959bda7d8fb9bde889e34a09dc03"[[package]]name = "cxxbridge-macro"-version = "1.0.86"+version = "1.0.90"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "39e61fda7e62115119469c7b3591fd913ecca96fb766cfd3f2e2502ab7bc87a5"+checksum = "ebf883b7aacd7b2aeb2a7b338648ee19f57c140d4ee8e52c68979c6b2f7f2263"dependencies = ["proc-macro2","quote",@@ -1048,9 +1049,9 @@ dependencies = [[[package]]name = "either"-version = "1.8.0"+version = "1.8.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797"+checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91"[[package]]name = "ena"@@ -1069,9 +1070,9 @@ checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"[[package]]name = "encoding_rs"-version = "0.8.31"+version = "0.8.32"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b"+checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394"dependencies = ["cfg-if",]@@ -1087,9 +1088,9 @@ dependencies = [[[package]]name = "exr"-version = "1.5.2"+version = "1.5.3"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8eb5f255b5980bb0c8cf676b675d1a99be40f316881444f44e0462eaf5df5ded"+checksum = "e8af5ef47e2ed89d23d0ecbc1b681b30390069de70260937877514377fc24feb"dependencies = ["bit_field","flume",@@ -1098,6 +1099,7 @@ dependencies = ["miniz_oxide","smallvec","threadpool",+ "zune-inflate",][[package]]@@ -1141,14 +1143,14 @@ dependencies = [[[package]]name = "filetime"-version = "0.2.19"+version = "0.2.20"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4e884668cd0c7480504233e951174ddc3b382f7c2666e3b7310b5c4e7b0c37f9"+checksum = "8a3de6e8d11b22ff9edc6d916f890800597d60f8b2da1caf2955c274638d6412"dependencies = ["cfg-if","libc","redox_syscall",- "windows-sys",+ "windows-sys 0.45.0",][[package]]@@ -1177,7 +1179,7 @@ dependencies = ["futures-sink","nanorand","pin-project",- "spin 0.9.4",+ "spin 0.9.5",][[package]]@@ -1247,9 +1249,9 @@ dependencies = [[[package]]name = "futures"-version = "0.3.25"+version = "0.3.26"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "38390104763dc37a5145a53c29c63c1290b5d316d6086ec32c293f6736051bb0"+checksum = "13e2792b0ff0340399d58445b88fd9770e3489eff258a4cbc1523418f12abf84"dependencies = ["futures-channel","futures-core",@@ -1262,9 +1264,9 @@ dependencies = [[[package]]name = "futures-channel"-version = "0.3.25"+version = "0.3.26"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed"+checksum = "2e5317663a9089767a1ec00a487df42e0ca174b61b4483213ac24448e4664df5"dependencies = ["futures-core","futures-sink",@@ -1272,15 +1274,15 @@ dependencies = [[[package]]name = "futures-core"-version = "0.3.25"+version = "0.3.26"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac"+checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608"[[package]]name = "futures-executor"-version = "0.3.25"+version = "0.3.26"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7acc85df6714c176ab5edf386123fafe217be88c0840ec11f199441134a074e2"+checksum = "e8de0a35a6ab97ec8869e32a2473f4b1324459e14c29275d14b10cb1fd19b50e"dependencies = ["futures-core","futures-task",@@ -1289,15 +1291,15 @@ dependencies = [[[package]]name = "futures-io"-version = "0.3.25"+version = "0.3.26"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb"+checksum = "bfb8371b6fb2aeb2d280374607aeabfc99d95c72edfe51692e42d3d7f0d08531"[[package]]name = "futures-macro"-version = "0.3.25"+version = "0.3.26"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "bdfb8ce053d86b91919aad980c220b1fb8401a9394410e1c289ed7e66b61835d"+checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70"dependencies = ["proc-macro2","quote",@@ -1306,21 +1308,21 @@ dependencies = [[[package]]name = "futures-sink"-version = "0.3.25"+version = "0.3.26"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9"+checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364"[[package]]name = "futures-task"-version = "0.3.25"+version = "0.3.26"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea"+checksum = "dcf79a1bf610b10f42aea489289c5a2c478a786509693b80cd39c44ccd936366"[[package]]name = "futures-util"-version = "0.3.25"+version = "0.3.26"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6"+checksum = "9c1d6de3acfef38d2be4b1f543f553131788603495be83da675e180c8d6b7bd1"dependencies = ["futures-channel","futures-core",@@ -1451,9 +1453,9 @@ dependencies = [[[package]]name = "heck"-version = "0.4.0"+version = "0.4.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9"+checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"[[package]]name = "hermit-abi"@@ -1566,9 +1568,9 @@ checksum = "02296996cb8796d7c6e3bc2d9211b7802812d36999a51bb754123ead7d37d026"[[package]]name = "hyper"-version = "0.14.23"+version = "0.14.24"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "034711faac9d2166cb1baf1a2fb0b60b1f277f8492fd72176c17f3515e1abd3c"+checksum = "5e011372fa0b68db8350aa7a248930ecc7839bf46d8485577d69f117a75f164c"dependencies = ["bytes","futures-channel",@@ -1787,9 +1789,9 @@ dependencies = [[[package]]name = "js-sys"-version = "0.3.60"+version = "0.3.61"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47"+checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730"dependencies = ["wasm-bindgen",]@@ -1880,9 +1882,9 @@ dependencies = [[[package]]name = "librocksdb-sys"-version = "0.8.0+7.4.4"+version = "0.8.3+7.4.4"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "611804e4666a25136fcc5f8cf425ab4d26c7f74ea245ffe92ea23b85b6420b5d"+checksum = "557b255ff04123fcc176162f56ed0c9cd42d8f357cf55b3fabeb60f7413741b3"dependencies = ["bindgen","bzip2-sys",@@ -2037,9 +2039,9 @@ dependencies = [[[package]]name = "matches"-version = "0.1.9"+version = "0.1.10"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f"+checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5"[[package]]name = "matchit"@@ -2163,7 +2165,7 @@ dependencies = ["libc","log","wasi 0.11.0+wasi-snapshot-preview1",- "windows-sys",+ "windows-sys 0.42.0",][[package]]@@ -2249,9 +2251,9 @@ dependencies = [[[package]]name = "num-complex"-version = "0.4.2"+version = "0.4.3"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7ae39348c8bc5fbd7f40c727a9925f03517afd2ab27d46702108b6a7e5414c19"+checksum = "02e0d21255c828d6f128a1e41534206671e8c3ea0c62f32291e808dc82cff17d"dependencies = ["num-traits",]@@ -2348,7 +2350,7 @@ dependencies = [[[package]]name = "onnxruntime"version = "0.0.14"-source = "git+https://github.com/Cuely/onnxruntime-rs#b7b48c9a6d4a553572caf8f1a595c3721a923a4d"+source = "git+https://github.com/Cuely/onnxruntime-rs#b8d281e0e6f1e180700e8df437f6a07a36fec931"dependencies = ["lazy_static","ndarray",@@ -2360,7 +2362,7 @@ dependencies = [[[package]]name = "onnxruntime-sys"version = "0.0.14"-source = "git+https://github.com/Cuely/onnxruntime-rs#b7b48c9a6d4a553572caf8f1a595c3721a923a4d"+source = "git+https://github.com/Cuely/onnxruntime-rs#b8d281e0e6f1e180700e8df437f6a07a36fec931"dependencies = ["flate2","tar",@@ -2470,15 +2472,15 @@ dependencies = [[[package]]name = "parking_lot_core"-version = "0.9.6"+version = "0.9.7"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ba1ef8814b5c993410bb3adfad7a5ed269563e4a2f90c41f5d85be7fb47133bf"+checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521"dependencies = ["cfg-if","libc","redox_syscall","smallvec",- "windows-sys",+ "windows-sys 0.45.0",][[package]]@@ -2507,9 +2509,9 @@ checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e"[[package]]name = "petgraph"-version = "0.6.2"+version = "0.6.3"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "e6d5014253a1331579ce62aa67443b4a658c5e7dd03d4bc6d302b94474888143"+checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4"dependencies = ["fixedbitset","indexmap",@@ -2716,9 +2718,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068"[[package]]name = "proc-macro2"-version = "1.0.49"+version = "1.0.51"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5"+checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6"dependencies = ["unicode-ident",]@@ -2857,9 +2859,9 @@ dependencies = [[[package]]name = "rayon-core"-version = "1.10.1"+version = "1.10.2"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "cac410af5d00ab6884528b4ab69d1e8e146e8d471201800fa1b4524126de6ad3"+checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b"dependencies = ["crossbeam-channel","crossbeam-deque",@@ -2924,11 +2926,11 @@ dependencies = [[[package]]name = "reqwest"-version = "0.11.13"+version = "0.11.14"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "68cc60575865c7831548863cc02356512e3f1dc2f3f82cb837d7fc4cc8f3c97c"+checksum = "21eed90ec8570952d53b772ecf8f206aa1ec9a3d76b2521c56c42973f2d91ee9"dependencies = [- "base64 0.13.1",+ "base64 0.21.0","bytes","encoding_rs","futures-core",@@ -2956,6 +2958,7 @@ dependencies = ["url","wasm-bindgen","wasm-bindgen-futures",+ "wasm-streams","web-sys","winreg",]@@ -3152,7 +3155,7 @@ version = "0.1.21"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3"dependencies = [- "windows-sys",+ "windows-sys 0.42.0",][[package]]@@ -3191,9 +3194,9 @@ dependencies = [[[package]]name = "security-framework"-version = "2.7.0"+version = "2.8.2"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "2bc1bb97804af6631813c55739f771071e0f2ed33ee20b68c86ec505d906356c"+checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254"dependencies = ["bitflags","core-foundation",@@ -3204,9 +3207,9 @@ dependencies = [[[package]]name = "security-framework-sys"-version = "2.6.1"+version = "2.8.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556"+checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4"dependencies = ["core-foundation-sys","libc",@@ -3270,9 +3273,9 @@ dependencies = [[[package]]name = "serde_json"-version = "1.0.91"+version = "1.0.93"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883"+checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76"dependencies = ["itoa 1.0.5","ryu",@@ -3349,6 +3352,12 @@ dependencies = ["libc",]+[[package]]+name = "simd-adler32"+version = "0.3.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "14a5df39617d7c8558154693a1bb8157a4aab8179209540cc0b10e5dc24e0b18"+[[package]]name = "siphasher"version = "0.3.10"@@ -3388,9 +3397,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"[[package]]name = "spin"-version = "0.9.4"+version = "0.9.5"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7f6002a767bff9e83f8eeecf883ecb8011875a21ae8da43bffb817a57e78cc09"+checksum = "7dccf47db1b41fa1573ed27ccf5e08e3ca771cb994f776668c5ebda893b248fc"dependencies = ["lock_api",]@@ -3476,9 +3485,9 @@ dependencies = [[[package]]name = "sync_wrapper"-version = "0.1.1"+version = "0.1.2"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8"+checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"[[package]]name = "tantivy"@@ -3766,9 +3775,9 @@ dependencies = [[[package]]name = "tinyvec_macros"-version = "0.1.0"+version = "0.1.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"+checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"[[package]]name = "tokenizers"@@ -3807,9 +3816,9 @@ dependencies = [[[package]]name = "tokio"-version = "1.24.1"+version = "1.25.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1d9f76183f91ecfb55e1d7d5602bd1d979e38a3a522fe900241cf195624d67ae"+checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af"dependencies = ["autocfg","bytes",@@ -3822,7 +3831,7 @@ dependencies = ["signal-hook-registry","socket2","tokio-macros",- "windows-sys",+ "windows-sys 0.42.0",][[package]]@@ -3838,9 +3847,9 @@ dependencies = [[[package]]name = "tokio-native-tls"-version = "0.3.0"+version = "0.3.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b"+checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"dependencies = ["native-tls","tokio",@@ -3859,9 +3868,9 @@ dependencies = [[[package]]name = "tokio-util"-version = "0.7.4"+version = "0.7.6"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740"+checksum = "bc6a3b08b64e6dfad376fa2432c7b1f01522e37a623c3050bc95db2d3ff21583"dependencies = ["bytes","futures-core",@@ -3873,9 +3882,9 @@ dependencies = [[[package]]name = "toml"-version = "0.5.10"+version = "0.5.11"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1333c76748e868a4d9d1017b5ab53171dfd095f70c712fdb4653a406547f598f"+checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234"dependencies = ["serde",]@@ -4020,9 +4029,9 @@ dependencies = [[[package]]name = "unicode-bidi"-version = "0.3.8"+version = "0.3.10"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992"+checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58"[[package]]name = "unicode-ident"@@ -4050,9 +4059,9 @@ dependencies = [[[package]]name = "unicode-segmentation"-version = "1.10.0"+version = "1.10.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0fdbf052a0783de01e944a6ce7a8cb939e295b1e7be835a1112c3b9a7f047a5a"+checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36"[[package]]name = "unicode-width"@@ -4080,9 +4089,9 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"[[package]]name = "ureq"-version = "2.6.1"+version = "2.6.2"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "733b5ad78377302af52c0dbcb2623d78fe50e4b3bf215948ff29e9ee031d8566"+checksum = "338b31dd1314f68f3aabf3ed57ab922df95ffcd902476ca7ba3c4ce7b908c46d"dependencies = ["base64 0.13.1","flate2",@@ -4131,9 +4140,9 @@ checksum = "5190c9442dcdaf0ddd50f37420417d219ae5261bbf5db120d0f9bab996c9cba1"[[package]]name = "uuid"-version = "1.2.2"+version = "1.3.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "422ee0de9031b5b948b97a8fc04e3aa35230001a722ddd27943e0be31564ce4c"+checksum = "1674845326ee10d37ca60470760d4288a6f80f304007d92e5c53bab78c9cfd79"dependencies = ["getrandom 0.2.8","serde",@@ -4204,9 +4213,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"[[package]]name = "wasm-bindgen"-version = "0.2.83"+version = "0.2.84"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268"+checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b"dependencies = ["cfg-if","wasm-bindgen-macro",@@ -4214,9 +4223,9 @@ dependencies = [[[package]]name = "wasm-bindgen-backend"-version = "0.2.83"+version = "0.2.84"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142"+checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9"dependencies = ["bumpalo","log",@@ -4229,9 +4238,9 @@ dependencies = [[[package]]name = "wasm-bindgen-futures"-version = "0.4.33"+version = "0.4.34"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d"+checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454"dependencies = ["cfg-if","js-sys",@@ -4241,9 +4250,9 @@ dependencies = [[[package]]name = "wasm-bindgen-macro"-version = "0.2.83"+version = "0.2.84"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810"+checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5"dependencies = ["quote","wasm-bindgen-macro-support",@@ -4251,9 +4260,9 @@ dependencies = [[[package]]name = "wasm-bindgen-macro-support"-version = "0.2.83"+version = "0.2.84"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c"+checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6"dependencies = ["proc-macro2","quote",@@ -4264,15 +4273,28 @@ dependencies = [[[package]]name = "wasm-bindgen-shared"-version = "0.2.83"+version = "0.2.84"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d"++[[package]]+name = "wasm-streams"+version = "0.2.3"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f"+checksum = "6bbae3363c08332cadccd13b67db371814cd214c2524020932f0804b8cf7c078"+dependencies = [+ "futures-util",+ "js-sys",+ "wasm-bindgen",+ "wasm-bindgen-futures",+ "web-sys",+][[package]]name = "web-sys"-version = "0.3.60"+version = "0.3.61"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f"+checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97"dependencies = ["js-sys","wasm-bindgen",@@ -4372,6 +4394,30 @@ dependencies = ["windows_x86_64_msvc 0.42.1",]+[[package]]+name = "windows-sys"+version = "0.45.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"+dependencies = [+ "windows-targets",+]++[[package]]+name = "windows-targets"+version = "0.42.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7"+dependencies = [+ "windows_aarch64_gnullvm",+ "windows_aarch64_msvc 0.42.1",+ "windows_i686_gnu 0.42.1",+ "windows_i686_msvc 0.42.1",+ "windows_x86_64_gnu 0.42.1",+ "windows_x86_64_gnullvm",+ "windows_x86_64_msvc 0.42.1",+]+[[package]]name = "windows_aarch64_gnullvm"version = "0.42.1"@@ -4508,11 +4554,20 @@ dependencies = [[[package]]name = "zstd-sys"-version = "2.0.5+zstd.1.5.2"+version = "2.0.7+zstd.1.5.4"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "edc50ffce891ad571e9f9afe5039c4837bede781ac4bb13052ed7ae695518596"+checksum = "94509c3ba2fe55294d752b79842c530ccfab760192521df74a081a78d2b3c7f5"dependencies = ["cc","libc","pkg-config",]++[[package]]+name = "zune-inflate"+version = "0.2.42"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c473377c11c4a3ac6a2758f944cd336678e9c977aa0abf54f6450cf77e902d6d"+dependencies = [+ "simd-adler32",+]diff --git a/core/src/entrypoint/entity.rs b/core/src/entrypoint/entity.rsindex af3334bd..e584404d 100644--- a/core/src/entrypoint/entity.rs+++ b/core/src/entrypoint/entity.rs@@ -113,7 +113,6 @@ impl EntityIndexer {let reader = BufReader::new(MultiBzDecoder::new(reader));let mut index = EntityIndex::open(output_path)?;EntityIterator::from(reader)- .into_iter().filter(|entity| !entity.categories.is_empty()).take(200_000).for_each(|entity| {diff --git a/core/src/query/parser.rs b/core/src/query/parser.rsindex ce6beddf..26410db1 100644--- a/core/src/query/parser.rs+++ b/core/src/query/parser.rs@@ -207,7 +207,6 @@ impl Term {fields.iter().filter(|(field, _)| ALL_FIELDS[field.field_id() as usize].is_searchable())- .into_iter().map(|(field, entry)| {(Occur::Should,@@ -232,7 +231,6 @@ impl Term {| Field::Text(TextField::SiteNoTokenizer))})- .into_iter().map(|(field, entry)| {(Occur::Should,diff --git a/core/src/spell/dictionary.rs b/core/src/spell/dictionary.rsindex 5cbf8cce..6acd5551 100644--- a/core/src/spell/dictionary.rs+++ b/core/src/spell/dictionary.rs@@ -268,7 +268,6 @@ impl<const TOP_N: usize> Dictionary<TOP_N> {self.cache.entry(term.chars()- .into_iter().map(|c| c.to_ascii_lowercase()).filter(|c| !matches!(c, ',' | '.' | '\\' | '=' | '*' | '(' | ')')).collect(),diff --git a/core/src/summarizer.rs b/core/src/summarizer.rsindex f857f164..37ced79d 100644--- a/core/src/summarizer.rs+++ b/core/src/summarizer.rs@@ -314,9 +314,70 @@ impl AbstractiveModel {}pub fn summarize(&self, text: &str, config: GenerationConfig) -> Result<String> {- let encoded_text = self.tokenizer.encode(text, true)?;+ let generator = BeamGenerator {+ model: self,+ config,+ state: Some(BeamState::Encoding {+ text: text.to_string(),+ }),+ };++ Ok(generator.collect())+ }+}++struct BeamGenerator<'a> {+ model: &'a AbstractiveModel,+ config: GenerationConfig,+ state: Option<BeamState>,+}++impl<'a> Iterator for BeamGenerator<'a> {+ type Item = String;++ fn next(&mut self) -> Option<Self::Item> {+ self.state+ .take()+ .and_then(|state| match state.step(self.model, &self.config) {+ Ok((state, res)) => {+ self.state = Some(state);+ res+ }+ Err(err) => {+ tracing::error!("Encountered an error while generating summary: {err}");+ None+ }+ })+ }+}++enum BeamState {+ Encoding {+ text: String,+ },+ Decoding {+ encoder_hidden_states: ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>,+ attention_mask: ArrayBase<OwnedRepr<i64>, Dim<IxDynImpl>>,+ },+ DecodingWithPast {+ encoder_hidden_states: ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>,+ attention_mask: ArrayBase<OwnedRepr<i64>, Dim<IxDynImpl>>,+ caches: Vec<ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>>,+ beams: Vec<Beam>,+ generated_tokens: u32,+ /// all beams agree on the tokens up to this index+ beam_token_agreement_idx: usize,+ },+ Remaining {+ terms: VecDeque<String>,+ },+}++impl BeamState {+ fn encode_step(model: &AbstractiveModel, text: String) -> Result<Self> {+ let encoded_text = model.tokenizer.encode(text.as_str(), true)?;- let mut ids = encoded_text+ let ids = encoded_text.get_ids().iter().map(|i| *i as i64)@@ -332,292 +393,492 @@ impl AbstractiveModel {let encoder_num_tokens = ids.len();- let mut decoder = self- .decoder- .lock()- .expect("Failed to get lock. Maybe another thread crashed?");+ let encoder_hidden_states = model.encode(ids, attention_mask.clone())?;- let mut decoder_with_past = self- .decoder_with_past+ let attention_mask: ArrayBase<OwnedRepr<i64>, Dim<IxDynImpl>> =+ ArrayBase::from_vec(attention_mask)+ .into_shape((1, encoder_num_tokens))+ .unwrap()+ .into_dyn();++ Ok(BeamState::Decoding {+ encoder_hidden_states,+ attention_mask,+ })+ }++ fn run_decoder_model(+ model: &AbstractiveModel,+ ids: &[i64],+ encoder_hidden_states: ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>,+ attention_mask: ArrayBase<OwnedRepr<i64>, Dim<IxDynImpl>>,+ ) -> Result<Vec<ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>>> {+ let mut decoder = model+ .decoder.lock().expect("Failed to get lock. Maybe another thread crashed?");- let arr_encoder_hidden_states = self.encode(ids.clone(), attention_mask.clone())?;-- let arr_attention_mask = ArrayBase::from_vec(attention_mask.clone())- .into_shape((1, encoder_num_tokens))+ let num_tokens = ids.len();+ let arr_ids = ArrayBase::from_vec(ids.to_vec())+ .into_shape((1, num_tokens)).unwrap().into_dyn();- let mut beams = Vec::new();-- ids = vec![self.begin_decoder_token as i64, self.bos_token_id as i64];- let mut generated_tokens = 0;+ let decoder_input = vec![+ TypedArray::I64(attention_mask),+ TypedArray::I64(arr_ids),+ TypedArray::F32(encoder_hidden_states),+ ];- let mut caches = Vec::new();+ let output = decoder.run(decoder_input)?;- loop {- generated_tokens += 1;+ Ok(output.into_iter().map(|t| t.clone().into_owned()).collect())+ }- if beams.is_empty() {- let num_tokens = ids.len();- let arr_ids = ArrayBase::from_vec(ids.clone())- .into_shape((1, num_tokens))- .unwrap()- .into_dyn();+ fn create_beams(+ decoder_output: &[ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>],+ ids: Vec<i64>,+ num_beams: u32,+ bos_token: usize,+ ) -> Vec<Beam> {+ let mut next_scores = BinaryHeap::with_capacity(num_beams as usize + 1);++ let all_logits = &decoder_output[0];+ let vocab_size = all_logits.shape()[2];+ let mut scores = Vec::with_capacity(vocab_size);++ for token_id in 0..vocab_size {+ let score: f32 = all_logits[[+ all_logits.shape()[0] - 1,+ all_logits.shape()[1] - 1,+ token_id,+ ]];+ scores.push(score);+ }- let decoder_input = vec![- TypedArray::I64(arr_attention_mask.clone()),- TypedArray::I64(arr_ids),- TypedArray::F32(arr_encoder_hidden_states.clone()),- ];+ softmax(&mut scores);- let decoder_output = decoder.run(decoder_input)?;- let mut next_scores = BinaryHeap::with_capacity(config.num_beams as usize + 1);+ for (token_id, score) in scores.into_iter().enumerate() {+ if token_id == bos_token {+ continue;+ }- let all_logits = &decoder_output[0];- let vocab_size = all_logits.shape()[2];- let mut scores = Vec::with_capacity(vocab_size);+ let scored_token = Reverse(ScoredToken {+ token: token_id as i64,+ score: score.log2(),+ });- for token_id in 0..vocab_size {- let score: f32 = all_logits[[0, num_tokens - 1, token_id]];- scores.push(score);+ if next_scores.len() < num_beams as usize {+ next_scores.push(scored_token);+ } else if let Some(mut worst) = next_scores.peek_mut() {+ if worst.0.score < scored_token.0.score {+ *worst = scored_token}+ }+ }- softmax(&mut scores);+ let mut beams = Vec::new();+ for scored_token in next_scores.into_iter().map(|t| t.0) {+ let mut ids = ids.clone();+ ids.push(scored_token.token);++ beams.push(Beam {+ input_ids: ids,+ score: scored_token.score,+ })+ }- for (token_id, score) in scores.into_iter().enumerate() {- if token_id == self.bos_token_id {- continue;- }+ beams+ }- let scored_token = Reverse(ScoredToken {- token: token_id as i64,- score: score.log2(),- });-- if next_scores.len() < config.num_beams as usize {- next_scores.push(scored_token);- } else if let Some(mut worst) = next_scores.peek_mut() {- if worst.0.score < scored_token.0.score {- *worst = scored_token- }- }+ fn create_caches(+ decoder_output: Vec<ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>>,+ num_beams: u32,+ ) -> Vec<ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>> {+ let num_outputs = decoder_output.len();++ decoder_output+ .into_iter()+ .map(|t| t.into_owned())+ .map(|mut t| {+ let orig = t.clone();++ for _ in 0..num_beams - 1 {+ t.append(Axis(0), orig.view()).unwrap();}- let num_outputs = decoder_output.len();+ t+ })+ .skip(1)+ .take(num_outputs - 2) // skip 'encoder_hidden_states' since this is not changed by the decoder model+ .collect::<Vec<_>>()+ }- caches = decoder_output- .into_iter()- .map(|t| t.clone().into_owned())- .map(|mut t| {- let orig = t.clone();+ fn run_decoder_model_with_past(+ model: &AbstractiveModel,+ beams: &[Beam],+ encoder_hidden_states: &ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>,+ attention_mask: &ArrayBase<OwnedRepr<i64>, Dim<IxDynImpl>>,+ mut caches: Vec<ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>>,+ ) -> Result<Vec<ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>>> {+ let mut decoder_with_past = model+ .decoder_with_past+ .lock()+ .expect("Failed to get lock. Maybe another thread crashed?");- for _ in 0..config.num_beams - 1 {- t.append(Axis(0), orig.view()).unwrap();- }+ let mut combined_attention_mask = Vec::new();+ let mut combined_encoder_hidden_states = Vec::with_capacity(beams.len());+ let mut combined_ids = Vec::new();- t- })- .skip(1)- .take(num_outputs - 2) // skip 'encoder_hidden_states' since this is not changed by the decoder model- .collect::<Vec<_>>();+ // TODO: Only advance non-finished beams- for scored_token in next_scores.into_iter().map(|t| t.0) {- let mut ids = ids.clone();- ids.push(scored_token.token);+ for beam in beams {+ combined_ids.push(*beam.input_ids.last().unwrap());- beams.push(Beam {- input_ids: ids,- score: scored_token.score,- })- }- } else {- let mut combined_attention_mask = Vec::new();- let mut combined_encoder_hidden_states = Vec::with_capacity(beams.len());- let mut combined_ids = Vec::new();+ for mask in attention_mask {+ combined_attention_mask.push(*mask);+ }- // TODO: Only advance non-finished beams+ for state in encoder_hidden_states {+ combined_encoder_hidden_states.push(*state);+ }+ }- for beam in &beams {- combined_ids.push(*beam.input_ids.last().unwrap());+ let combined_ids = ArrayBase::<OwnedRepr<_>, _>::from_vec(combined_ids)+ .into_shape((beams.len(), 1))+ .unwrap()+ .into_dyn();- for mask in &attention_mask {- combined_attention_mask.push(*mask);- }+ let mut attention_shape = attention_mask.shape().to_vec();+ attention_shape[0] = beams.len();- for state in &arr_encoder_hidden_states {- combined_encoder_hidden_states.push(*state);- }- }+ let combined_attention_mask =+ ArrayBase::<OwnedRepr<_>, _>::from_vec(combined_attention_mask)+ .into_shape(attention_shape.as_slice())+ .unwrap();- let combined_ids = ArrayBase::<OwnedRepr<_>, _>::from_vec(combined_ids)- .into_shape((beams.len(), 1))- .unwrap()- .into_dyn();+ let mut state_shape = encoder_hidden_states.shape().to_vec();+ state_shape[0] = beams.len();- let mut attention_shape = arr_attention_mask.shape().to_vec();- attention_shape[0] = beams.len();+ let combined_encoder_hidden_states =+ ArrayBase::<OwnedRepr<_>, _>::from_vec(combined_encoder_hidden_states)+ .into_shape(state_shape.as_slice())+ .unwrap();- let combined_attention_mask =- ArrayBase::<OwnedRepr<_>, _>::from_vec(combined_attention_mask)- .into_shape(attention_shape.as_slice())- .unwrap();+ let mut decoder_input = vec![+ TypedArray::I64(combined_attention_mask),+ TypedArray::I64(combined_ids),+ TypedArray::F32(combined_encoder_hidden_states),+ ];- let mut state_shape = arr_encoder_hidden_states.shape().to_vec();- state_shape[0] = beams.len();+ for tensor in caches.drain(0..) {+ decoder_input.push(TypedArray::F32(tensor));+ }- let combined_encoder_hidden_states =- ArrayBase::<OwnedRepr<_>, _>::from_vec(combined_encoder_hidden_states)- .into_shape(state_shape.as_slice())- .unwrap();+ let res = decoder_with_past.run(decoder_input)?;- let mut decoder_input = vec![- TypedArray::I64(combined_attention_mask),- TypedArray::I64(combined_ids),- TypedArray::F32(combined_encoder_hidden_states),- ];+ Ok(res.into_iter().map(|t| t.clone().into_owned()).collect())+ }- for tensor in caches.drain(0..) {- decoder_input.push(TypedArray::F32(tensor));- }+ fn new_possible_beams(+ decoder_output: &[ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>],+ beams: &[Beam],+ force_min_tokens: Option<u32>,+ generated_tokens: u32,+ eos_token_id: usize,+ length_penalty: f32,+ ) -> Vec<PossibleBeam> {+ let all_logits = &decoder_output[0];+ let vocab_size = all_logits.shape()[2];++ let mut beam_tok_scores = Vec::with_capacity(beams.len());+ for beam_idx in 0..beams.len() {+ let mut scores = Vec::with_capacity(vocab_size);++ for token_id in 0..vocab_size {+ scores.push(all_logits[[beam_idx, 0, token_id]]);+ }- let decoder_output: Vec<OrtOwnedTensor<f32, Dim<IxDynImpl>>> =- decoder_with_past.run(decoder_input)?;+ softmax(&mut scores);+ beam_tok_scores.push(scores)+ }- let all_logits = &decoder_output[0];- let vocab_size = all_logits.shape()[2];+ let mut new_beams = BinaryHeap::with_capacity(beams.len());- let mut beam_tok_scores = Vec::with_capacity(beams.len());- for beam_idx in 0..beams.len() {- let mut scores = Vec::with_capacity(vocab_size);+ let mut skip_eos = false;- for token_id in 0..vocab_size {- scores.push(all_logits[[beam_idx, 0, token_id]]);- }+ if let Some(min_tokens) = force_min_tokens {+ if generated_tokens < min_tokens {+ skip_eos = true;+ }+ }- softmax(&mut scores);- beam_tok_scores.push(scores)+ for beam_idx in 0..beams.len() {+ for new_token in 0..vocab_size {+ if skip_eos && new_token == eos_token_id {+ continue;}- let num_outputs = decoder_output.len();+ let tok_score = beam_tok_scores[beam_idx][new_token];- // skip(num_outputs-2) to skip 'encoder_hidden_states' since this is not changed by the decoder model- let output_caches: Vec<_> = decoder_output- .iter()- .skip(1)- .take(num_outputs - 2)- .collect();+ let tok_score = tok_score.log2();- let mut new_beams = BinaryHeap::with_capacity(beams.len());+ let new_score = beams[beam_idx].score + tok_score;- for beam_idx in 0..beams.len() {- for new_token in 0..vocab_size {- let tok_score = beam_tok_scores[beam_idx][new_token];+ let possible_beam = PossibleBeam {+ beam_idx,+ beam_length: beams[beam_idx].len(),+ new_token: new_token as i64,+ length_penalty,+ eos_token: eos_token_id as i64,+ new_score,+ };- let tok_score = tok_score.log2();+ if new_beams.len() < beams.len() {+ new_beams.push(Reverse(possible_beam));+ } else if let Some(mut worst) = new_beams.peek_mut() {+ if possible_beam > worst.0 {+ *worst = Reverse(possible_beam);+ }+ }+ }+ }- let new_score = beams[beam_idx].score + tok_score;+ new_beams.into_iter().map(|r| r.0).collect()+ }- let possible_beam = PossibleBeam {- beam_idx,- beam_length: beams[beam_idx].len(),- new_token: new_token as i64,- length_penalty: config.length_penalty,- eos_token: self.eos_token_id as i64,- new_score,- };+ fn new_caches(+ decoder_output: &[ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>],+ new_beams: &[PossibleBeam],+ ) -> Vec<ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>> {+ let num_outputs = decoder_output.len();+ // skip(num_outputs-2) to skip 'encoder_hidden_states' since this is not changed by the decoder model+ let output_caches: Vec<_> = decoder_output+ .iter()+ .skip(1)+ .take(num_outputs - 2)+ .collect();++ let mut new_caches = Vec::with_capacity(output_caches.len());++ for _ in &output_caches {+ new_caches.push(Vec::with_capacity(100));+ }- if new_beams.len() < beams.len() {- new_beams.push(Reverse(possible_beam));- } else if let Some(mut worst) = new_beams.peek_mut() {- if possible_beam > worst.0 {- *worst = Reverse(possible_beam);- }- }+ for possible_beam in new_beams {+ for (new_cache, old_cache) in new_caches.iter_mut().zip_eq(output_caches.iter()) {+ let shape = old_cache.shape();++ new_cache.push(+ old_cache+ .index_axis(Axis(0), possible_beam.beam_idx)+ .into_shape((1, shape[1], shape[2], shape[3]))+ .unwrap()+ .into_dyn(),+ );+ }+ }++ new_caches+ .into_iter()+ .map(|cache| onnxruntime::ndarray::concatenate(Axis(0), cache.as_slice()).unwrap())+ .collect()+ }++ fn cement_beams(+ beams: Vec<Beam>,+ possible_beams: Vec<PossibleBeam>,+ eos_token_id: i64,+ ) -> Vec<Beam> {+ possible_beams+ .into_iter()+ .map(|possible_beam| {+ if beams[possible_beam.beam_idx].is_finished(eos_token_id) {+ beams[possible_beam.beam_idx].clone()+ } else {+ let mut input_ids = beams[possible_beam.beam_idx].input_ids.clone();+ input_ids.push(possible_beam.new_token);+ Beam {+ input_ids,+ score: possible_beam.new_score,}}+ })+ .collect()+ }++ fn output_from_best(+ model: &AbstractiveModel,+ new_beams: Vec<Beam>,+ beam_agreement_index: usize,+ ) -> BeamState {+ let best_beam = new_beams+ .into_iter()+ .max_by(|a, b| a.score.total_cmp(&b.score))+ .unwrap();++ let terms: VecDeque<_> = best_beam+ .input_ids+ .into_iter()+ .skip(beam_agreement_index)+ .map(|i| i as u32)+ .map(|tok| model.tokenizer.decode(vec![tok], true).unwrap())+ .collect();++ BeamState::Remaining { terms }+ }- let mut new_caches = Vec::with_capacity(output_caches.len());+ fn check_early_stopping(early_stopping: Option<&EarlyStopping>, generated_tokens: u32) -> bool {+ early_stopping+ .map(|early_stopping| match early_stopping {+ EarlyStopping::MaxTokens { max_new_tokens } => generated_tokens >= *max_new_tokens,+ })+ .unwrap_or(false)+ }- for _ in &output_caches {- new_caches.push(Vec::with_capacity(100));+ fn all_beams_agree(beams: &[Beam], beam_token_agreement_idx: usize) -> bool {+ if beams+ .iter()+ .any(|beam| beam_token_agreement_idx >= beam.len())+ {+ return false;+ }++ beams+ .iter()+ .map(|beam| beam.input_ids[beam_token_agreement_idx])+ .all_equal()+ }++ fn step<'a>(+ mut self,+ model: &'a AbstractiveModel,+ config: &'a GenerationConfig,+ ) -> Result<(Self, Option<<BeamGenerator<'a> as Iterator>::Item>)> {+ loop {+ match self {+ BeamState::Encoding { text } => {+ self = Self::encode_step(model, text)?;}+ BeamState::Decoding {+ encoder_hidden_states,+ attention_mask,+ } => {+ let ids = vec![model.begin_decoder_token as i64, model.bos_token_id as i64];+ let decoder_output = Self::run_decoder_model(+ model,+ &ids,+ encoder_hidden_states.clone(),+ attention_mask.clone(),+ )?;++ let beams = Self::create_beams(+ &decoder_output,+ ids,+ config.num_beams,+ model.bos_token_id,+ );++ let caches = Self::create_caches(decoder_output, config.num_beams);++ self = BeamState::DecodingWithPast {+ encoder_hidden_states,+ attention_mask,+ caches,+ beams,+ generated_tokens: 1,+ beam_token_agreement_idx: 0,+ };+ }++ BeamState::DecodingWithPast {+ encoder_hidden_states,+ attention_mask,+ caches,+ mut beams,+ generated_tokens,+ beam_token_agreement_idx,+ } => {+ let shortest_beam = beams+ .iter()+ .min_by(|a, b| a.input_ids.len().cmp(&b.input_ids.len()))+ .unwrap();- for possible_beam in &new_beams {- for (new_cache, old_cache) in new_caches.iter_mut().zip_eq(output_caches.iter())+ if (shortest_beam.input_ids.len() - beam_token_agreement_idx) as u32+ > config.max_beam_divergence_tokens{- let shape = old_cache.shape();-- new_cache.push(- old_cache- .index_axis(Axis(0), possible_beam.0.beam_idx)- .into_shape((1, shape[1], shape[2], shape[3]))- .unwrap()- .into_dyn(),- );+ let longest_beam = beams+ .iter()+ .max_by(|a, b| a.input_ids.len().cmp(&b.input_ids.len()))+ .unwrap()+ .clone();++ let num_beams = beams.len();+ beams = vec![longest_beam; num_beams];}- }- let new_caches: Vec<_> = new_caches- .into_iter()- .map(|cache| {- onnxruntime::ndarray::concatenate(Axis(0), cache.as_slice()).unwrap()- })- .collect();-- let new_beams: Vec<_> = new_beams- .into_iter()- .map(|r| r.0)- .map(|possible_beam| {- if beams[possible_beam.beam_idx].is_finished(self.eos_token_id as i64) {- beams[possible_beam.beam_idx].clone()- } else {- let mut input_ids = beams[possible_beam.beam_idx].input_ids.clone();- input_ids.push(possible_beam.new_token);- Beam {- input_ids,- score: possible_beam.new_score,- }- }- })- .collect();-- beams = new_beams;- caches = new_caches;-- if beams- .iter()- .all(|beam| beam.is_finished(self.eos_token_id as i64))- {- break;- }- }+ if Self::all_beams_agree(&beams, beam_token_agreement_idx) {+ let tok = beams[0].input_ids[beam_token_agreement_idx];+ let decoded = model.tokenizer.decode(vec![tok as u32], true)?;++ self = BeamState::DecodingWithPast {+ encoder_hidden_states,+ attention_mask,+ caches,+ beams,+ generated_tokens,+ beam_token_agreement_idx: beam_token_agreement_idx + 1,+ };- if let Some(min_tokens) = config.force_min_tokens {- if generated_tokens < min_tokens {- continue;- }- }+ return Ok((self, Some(decoded)));+ }- if let Some(early_stopping) = &config.early_stopping {- match early_stopping {- EarlyStopping::MaxTokens { max_new_tokens } => {- if generated_tokens >= *max_new_tokens {- break;- }+ let decoder_output = Self::run_decoder_model_with_past(+ model,+ &beams,+ &encoder_hidden_states,+ &attention_mask,+ caches,+ )?;++ let possible_beams = Self::new_possible_beams(+ &decoder_output,+ &beams,+ config.force_min_tokens,+ generated_tokens,+ model.eos_token_id,+ config.length_penalty,+ );++ let new_caches = Self::new_caches(&decoder_output, &possible_beams);++ let new_beams =+ Self::cement_beams(beams, possible_beams, model.eos_token_id as i64);++ if Self::check_early_stopping(config.early_stopping.as_ref(), generated_tokens)+ || new_beams+ .iter()+ .all(|beam| beam.is_finished(model.eos_token_id as i64))+ {+ self = Self::output_from_best(model, new_beams, beam_token_agreement_idx);+ } else {+ self = BeamState::DecodingWithPast {+ encoder_hidden_states,+ attention_mask,+ caches: new_caches,+ beams: new_beams,+ generated_tokens: generated_tokens + 1,+ beam_token_agreement_idx,+ };}}+ BeamState::Remaining { mut terms } => {+ let term = terms.pop_front();++ return Ok((BeamState::Remaining { terms }, term));+ }}}-- let best_beam = beams- .into_iter()- .max_by(|a, b| a.score.total_cmp(&b.score))- .unwrap();-- let res: Vec<_> = best_beam.input_ids.into_iter().map(|i| i as u32).collect();- Ok(self.tokenizer.decode(res, true).unwrap())}}@@ -716,8 +977,13 @@ impl Eq for PossibleBeam {}pub struct GenerationConfig {pub early_stopping: Option<EarlyStopping>,pub force_min_tokens: Option<u32>,+ /// Exponential penalty to the length that is used with beam-based generation.+ /// It is applied as an exponent to the sequence length, which in turn is used to divide the score of the sequence.+ /// Since the score is the log likelihood of the sequence (i.e. negative),+ /// length_penalty > 0.0 promotes longer sequences, while length_penalty < 0.0 encourages shorter sequencespub length_penalty: f32,pub num_beams: u32,+ pub max_beam_divergence_tokens: u32,}impl Default for GenerationConfig {@@ -729,6 +995,7 @@ impl Default for GenerationConfig {force_min_tokens: None,length_penalty: 1.0,num_beams: 5,+ max_beam_divergence_tokens: 20,}}}@@ -788,7 +1055,7 @@ mod tests {let config = GenerationConfig {num_beams: 2,length_penalty: 1.0,- early_stopping: Some(EarlyStopping::MaxTokens { max_new_tokens: 32 }),+ early_stopping: Some(EarlyStopping::MaxTokens { max_new_tokens: 16 }),..Default::default()};@@ -796,6 +1063,7 @@ mod tests {let summary = model.summarize(text, config).unwrap();println!("Elapsed: {:?}", start.elapsed());println!("{:?}", &summary);+assert!(summary.len() > 50);assert!(summary.contains("Aristotle"));}diff --git a/core/src/webpage/region.rs b/core/src/webpage/region.rsindex edae215d..6852578e 100644--- a/core/src/webpage/region.rs+++ b/core/src/webpage/region.rs@@ -22,8 +22,9 @@ use crate::{Error, Result};use super::Webpage;-#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, Serialize, Deserialize)]+#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, Serialize, Deserialize, Default)]pub enum Region {+ #[default]All,Denmark,France,@@ -32,12 +33,6 @@ pub enum Region {US,}-impl Default for Region {- fn default() -> Self {- Region::All- }-}-pub const ALL_REGIONS: [Region; 6] = [Region::All,Region::Denmark,