Skip to content

web search done right

simpler keyphrase extraction

Mikkel Denkerebbfffdb8e1d
diff --git a/assets/licenses.html b/assets/licenses.html
index 1e67cd37..c42f1dda 100644
--- a/assets/licenses.html
+++ b/assets/licenses.html
@@ -44,11 +44,11 @@
<h2>Overview of licenses:</h2>
<ul class="licenses-overview">
- <li><a href="#Apache-2.0">Apache License 2.0</a> (421)</li>
- <li><a href="#MIT">MIT License</a> (187)</li>
+ <li><a href="#Apache-2.0">Apache License 2.0</a> (423)</li>
+ <li><a href="#MIT">MIT License</a> (186)</li>
<li><a href="#BSD-3-Clause">BSD 3-Clause &quot;New&quot; or &quot;Revised&quot; License</a> (9)</li>
<li><a href="#MPL-2.0">Mozilla Public License 2.0</a> (9)</li>
- <li><a href="#AGPL-3.0">GNU Affero General Public License v3.0</a> (6)</li>
+ <li><a href="#AGPL-3.0">GNU Affero General Public License v3.0</a> (7)</li>
<li><a href="#ISC">ISC License</a> (4)</li>
<li><a href="#Unicode-3.0">Unicode License v3</a> (4)</li>
<li><a href="#BSD-2-Clause">BSD 2-Clause &quot;Simplified&quot; License</a> (3)</li>
@@ -70,6 +70,7 @@
<ul class="license-used-by">
<li><a href=" https://crates.io/crates/bloom ">bloom 0.1.0</a></li>
<li><a href=" https://crates.io/crates/file_store ">file_store 0.1.0</a></li>
+ <li><a href=" https://crates.io/crates/leechy ">leechy 0.1.0</a></li>
<li><a href=" https://crates.io/crates/lending-iter ">lending-iter 0.1.0</a></li>
<li><a href=" https://crates.io/crates/ownedbytes ">ownedbytes 0.1.0</a></li>
<li><a href=" https://crates.io/crates/robotstxt ">robotstxt 0.1.0</a></li>
@@ -327,8 +328,8 @@ You should also get your employer (if you work as a programmer) or school, if an
<li><a href=" https://github.com/Actyx/sync_wrapper ">sync_wrapper 0.1.2</a></li>
<li><a href=" https://github.com/Actyx/sync_wrapper ">sync_wrapper 1.0.1</a></li>
<li><a href=" https://github.com/gyscos/zstd-rs ">zstd-safe 5.0.2+zstd.1.5.2</a></li>
- <li><a href=" https://github.com/gyscos/zstd-rs ">zstd-safe 7.2.0</a></li>
- <li><a href=" https://github.com/gyscos/zstd-rs ">zstd-sys 2.0.12+zstd.1.5.6</a></li>
+ <li><a href=" https://github.com/gyscos/zstd-rs ">zstd-safe 7.2.1</a></li>
+ <li><a href=" https://github.com/gyscos/zstd-rs ">zstd-sys 2.0.13+zstd.1.5.6</a></li>
</ul>
<pre class="license-text">
Apache License
@@ -1571,7 +1572,7 @@ You should also get your employer (if you work as a programmer) or school, if an
<h3 id="Apache-2.0">Apache License 2.0</h3>
<h4>Used by:</h4>
<ul class="license-used-by">
- <li><a href=" https://github.com/bytecodealliance/target-lexicon ">target-lexicon 0.12.15</a></li>
+ <li><a href=" https://github.com/bytecodealliance/target-lexicon ">target-lexicon 0.12.16</a></li>
</ul>
<pre class="license-text">
Apache License
@@ -1802,6 +1803,7 @@ Software.
<li><a href=" https://github.com/microsoft/windows-rs ">windows-core 0.52.0</a></li>
<li><a href=" https://github.com/microsoft/windows-rs ">windows-sys 0.48.0</a></li>
<li><a href=" https://github.com/microsoft/windows-rs ">windows-sys 0.52.0</a></li>
+ <li><a href=" https://github.com/microsoft/windows-rs ">windows-sys 0.59.0</a></li>
<li><a href=" https://github.com/microsoft/windows-rs ">windows-targets 0.48.5</a></li>
<li><a href=" https://github.com/microsoft/windows-rs ">windows-targets 0.52.6</a></li>
<li><a href=" https://github.com/microsoft/windows-rs ">windows_aarch64_gnullvm 0.48.5</a></li>
@@ -2237,6 +2239,7 @@ Software.
<h3 id="Apache-2.0">Apache License 2.0</h3>
<h4>Used by:</h4>
<ul class="license-used-by">
+ <li><a href=" https://github.com/google/zerocopy ">zerocopy-derive 0.7.35</a></li>
<li><a href=" https://github.com/google/zerocopy ">zerocopy 0.7.35</a></li>
</ul>
<pre class="license-text"> Apache License
@@ -2663,12 +2666,12 @@ Software.
<li><a href=" https://github.com/enarx/ciborium ">ciborium-io 0.2.2</a></li>
<li><a href=" https://github.com/enarx/ciborium ">ciborium-ll 0.2.2</a></li>
<li><a href=" https://github.com/enarx/ciborium ">ciborium 0.2.2</a></li>
- <li><a href=" https://github.com/clap-rs/clap ">clap_builder 4.5.10</a></li>
- <li><a href=" https://github.com/clap-rs/clap ">clap_derive 4.5.8</a></li>
- <li><a href=" https://github.com/clap-rs/clap ">clap_lex 0.7.1</a></li>
+ <li><a href=" https://github.com/clap-rs/clap ">clap_builder 4.5.14</a></li>
+ <li><a href=" https://github.com/clap-rs/clap ">clap_derive 4.5.13</a></li>
+ <li><a href=" https://github.com/clap-rs/clap ">clap_lex 0.7.2</a></li>
<li><a href=" https://github.com/Narsil/esaxx-rs ">esaxx-rs 0.1.10</a></li>
<li><a href=" https://github.com/cberner/redb ">redb 2.1.1</a></li>
- <li><a href=" https://github.com/huggingface/safetensors ">safetensors 0.4.3</a></li>
+ <li><a href=" https://github.com/huggingface/safetensors ">safetensors 0.4.4</a></li>
<li><a href=" https://github.com/huggingface/spm_precompiled ">spm_precompiled 0.1.4</a></li>
<li><a href=" https://github.com/huggingface/tokenizers ">tokenizers 0.13.4</a></li>
<li><a href=" https://github.com/cameron1024/unarray ">unarray 0.1.4</a></li>
@@ -3719,7 +3722,7 @@ Software.
<h3 id="Apache-2.0">Apache License 2.0</h3>
<h4>Used by:</h4>
<ul class="license-used-by">
- <li><a href=" https://github.com/rust-cli/anstyle.git ">anstyle-parse 0.2.4</a></li>
+ <li><a href=" https://github.com/rust-cli/anstyle.git ">anstyle-parse 0.2.5</a></li>
<li><a href=" https://github.com/phil-opp/rust-bit-field ">bit_field 0.10.2</a></li>
<li><a href=" https://github.com/llogiq/bytecount ">bytecount 0.6.8</a></li>
<li><a href=" https://github.com/utkarshkukreti/diff.rs ">diff 0.1.13</a></li>
@@ -3933,13 +3936,13 @@ Software.
<h3 id="Apache-2.0">Apache License 2.0</h3>
<h4>Used by:</h4>
<ul class="license-used-by">
- <li><a href=" https://github.com/rust-cli/anstyle.git ">anstream 0.6.14</a></li>
- <li><a href=" https://github.com/rust-cli/anstyle ">anstyle-query 1.1.0</a></li>
- <li><a href=" https://github.com/rust-cli/anstyle.git ">anstyle-wincon 3.0.3</a></li>
- <li><a href=" https://github.com/rust-cli/anstyle.git ">anstyle 1.0.7</a></li>
- <li><a href=" https://github.com/clap-rs/clap ">clap 4.5.10</a></li>
+ <li><a href=" https://github.com/rust-cli/anstyle.git ">anstream 0.6.15</a></li>
+ <li><a href=" https://github.com/rust-cli/anstyle ">anstyle-query 1.1.1</a></li>
+ <li><a href=" https://github.com/rust-cli/anstyle.git ">anstyle-wincon 3.0.4</a></li>
+ <li><a href=" https://github.com/rust-cli/anstyle.git ">anstyle 1.0.8</a></li>
+ <li><a href=" https://github.com/clap-rs/clap ">clap 4.5.14</a></li>
<li><a href=" https://github.com/jamesmunns/cobs.rs ">cobs 0.2.3</a></li>
- <li><a href=" https://github.com/rust-cli/anstyle ">colorchoice 1.0.1</a></li>
+ <li><a href=" https://github.com/rust-cli/anstyle ">colorchoice 1.0.2</a></li>
<li><a href=" https://github.com/srijs/rust-crc32fast ">crc32fast 1.4.2</a></li>
<li><a href=" https://github.com/colin-kiegel/rust-derive-builder ">derive_builder 0.12.0</a></li>
<li><a href=" https://github.com/colin-kiegel/rust-derive-builder ">derive_builder_core 0.12.0</a></li>
@@ -3948,18 +3951,18 @@ Software.
<li><a href=" https://github.com/sfackler/foreign-types ">foreign-types-shared 0.1.1</a></li>
<li><a href=" https://github.com/sfackler/foreign-types ">foreign-types 0.3.2</a></li>
<li><a href=" https://github.com/KokaKiwi/rust-hex ">hex 0.4.3</a></li>
- <li><a href=" https://github.com/polyfill-rs/is_terminal_polyfill ">is_terminal_polyfill 1.70.0</a></li>
+ <li><a href=" https://github.com/polyfill-rs/is_terminal_polyfill ">is_terminal_polyfill 1.70.1</a></li>
<li><a href=" https://github.com/sfackler/rust-native-tls ">native-tls 0.2.12</a></li>
<li><a href=" https://crates.io/crates/openssl-macros ">openssl-macros 0.1.1</a></li>
<li><a href=" https://github.com/rust-pretty-assertions/rust-pretty-assertions ">pretty_assertions 1.4.0</a></li>
<li><a href=" http://github.com/tailhook/quick-error ">quick-error 1.2.3</a></li>
<li><a href=" http://github.com/tailhook/quick-error ">quick-error 2.0.1</a></li>
- <li><a href=" https://github.com/toml-rs/toml ">serde_spanned 0.6.6</a></li>
- <li><a href=" https://github.com/toml-rs/toml ">toml 0.8.15</a></li>
- <li><a href=" https://github.com/toml-rs/toml ">toml_datetime 0.6.6</a></li>
+ <li><a href=" https://github.com/toml-rs/toml ">serde_spanned 0.6.7</a></li>
+ <li><a href=" https://github.com/toml-rs/toml ">toml 0.8.19</a></li>
+ <li><a href=" https://github.com/toml-rs/toml ">toml_datetime 0.6.8</a></li>
<li><a href=" https://github.com/toml-rs/toml ">toml_edit 0.19.15</a></li>
<li><a href=" https://github.com/toml-rs/toml ">toml_edit 0.21.1</a></li>
- <li><a href=" https://github.com/toml-rs/toml ">toml_edit 0.22.16</a></li>
+ <li><a href=" https://github.com/toml-rs/toml ">toml_edit 0.22.20</a></li>
<li><a href=" https://github.com/swgillespie/unicode-categories ">unicode_categories 0.1.1</a></li>
</ul>
<pre class="license-text"> Apache License
@@ -4758,9 +4761,9 @@ END OF TERMS AND CONDITIONS</pre>
<li><a href=" https://github.com/dtolnay/monostate ">monostate-impl 0.1.13</a></li>
<li><a href=" https://github.com/dtolnay/monostate ">monostate 0.1.13</a></li>
<li><a href=" https://github.com/illicitonion/num_enum ">num_enum 0.6.1</a></li>
- <li><a href=" https://github.com/illicitonion/num_enum ">num_enum 0.7.2</a></li>
+ <li><a href=" https://github.com/illicitonion/num_enum ">num_enum 0.7.3</a></li>
<li><a href=" https://github.com/illicitonion/num_enum ">num_enum_derive 0.6.1</a></li>
- <li><a href=" https://github.com/illicitonion/num_enum ">num_enum_derive 0.7.2</a></li>
+ <li><a href=" https://github.com/illicitonion/num_enum ">num_enum_derive 0.7.3</a></li>
<li><a href=" https://github.com/dtolnay/paste ">paste 1.0.15</a></li>
<li><a href=" https://github.com/dtolnay/proc-macro2 ">proc-macro2 1.0.86</a></li>
<li><a href=" https://github.com/dtolnay/quote ">quote 1.0.36</a></li>
@@ -4768,9 +4771,9 @@ END OF TERMS AND CONDITIONS</pre>
<li><a href=" https://github.com/dtolnay/ryu ">ryu 1.0.18</a></li>
<li><a href=" https://github.com/dtolnay/semver ">semver 1.0.23</a></li>
<li><a href=" https://github.com/dtolnay/seq-macro ">seq-macro 0.3.5</a></li>
- <li><a href=" https://github.com/serde-rs/serde ">serde 1.0.204</a></li>
- <li><a href=" https://github.com/serde-rs/serde ">serde_derive 1.0.204</a></li>
- <li><a href=" https://github.com/serde-rs/json ">serde_json 1.0.120</a></li>
+ <li><a href=" https://github.com/serde-rs/serde ">serde 1.0.205</a></li>
+ <li><a href=" https://github.com/serde-rs/serde ">serde_derive 1.0.205</a></li>
+ <li><a href=" https://github.com/serde-rs/json ">serde_json 1.0.122</a></li>
<li><a href=" https://github.com/dtolnay/path-to-error ">serde_path_to_error 0.1.16</a></li>
<li><a href=" https://github.com/nox/serde_urlencoded ">serde_urlencoded 0.7.1</a></li>
<li><a href=" https://github.com/dtolnay/syn ">syn 2.0.72</a></li>
@@ -6436,7 +6439,7 @@ limitations under the License.
<h3 id="Apache-2.0">Apache License 2.0</h3>
<h4>Used by:</h4>
<ul class="license-used-by">
- <li><a href=" https://github.com/cryptocorrosion/cryptocorrosion ">ppv-lite86 0.2.17</a></li>
+ <li><a href=" https://github.com/cryptocorrosion/cryptocorrosion ">ppv-lite86 0.2.20</a></li>
</ul>
<pre class="license-text"> Apache License
Version 2.0, January 2004
@@ -7273,7 +7276,7 @@ limitations under the License.
<h3 id="Apache-2.0">Apache License 2.0</h3>
<h4>Used by:</h4>
<ul class="license-used-by">
- <li><a href=" https://github.com/rustls/pki-types ">rustls-pki-types 1.7.0</a></li>
+ <li><a href=" https://github.com/rustls/pki-types ">rustls-pki-types 1.8.0</a></li>
</ul>
<pre class="license-text"> Apache License
Version 2.0, January 2004
@@ -7920,7 +7923,7 @@ limitations under the License.</pre>
<li><a href=" https://github.com/alexcrichton/bzip2-rs ">bzip2-sys 0.1.11+1.0.8</a></li>
<li><a href=" https://github.com/alexcrichton/bzip2-rs ">bzip2 0.4.4</a></li>
<li><a href=" https://github.com/japaric/cast.rs ">cast 0.3.0</a></li>
- <li><a href=" https://github.com/rust-lang/cc-rs ">cc 1.1.6</a></li>
+ <li><a href=" https://github.com/rust-lang/cc-rs ">cc 1.1.8</a></li>
<li><a href=" https://github.com/alexcrichton/cfg-if ">cfg-if 1.0.0</a></li>
<li><a href=" https://github.com/tkaitchuck/constrandom ">const-random-macro 0.1.16</a></li>
<li><a href=" https://github.com/tkaitchuck/constrandom ">const-random 0.1.18</a></li>
@@ -7941,9 +7944,9 @@ limitations under the License.</pre>
<li><a href=" https://github.com/cuviper/equivalent ">equivalent 1.0.1</a></li>
<li><a href=" https://github.com/lambda-fairy/rust-errno ">errno 0.3.9</a></li>
<li><a href=" https://github.com/smol-rs/fastrand ">fastrand 2.1.0</a></li>
- <li><a href=" https://github.com/alexcrichton/filetime ">filetime 0.2.23</a></li>
+ <li><a href=" https://github.com/alexcrichton/filetime ">filetime 0.2.24</a></li>
<li><a href=" https://github.com/petgraph/fixedbitset ">fixedbitset 0.4.2</a></li>
- <li><a href=" https://github.com/rust-lang/flate2-rs ">flate2 1.0.30</a></li>
+ <li><a href=" https://github.com/rust-lang/flate2-rs ">flate2 1.0.31</a></li>
<li><a href=" https://github.com/servo/rust-fnv ">fnv 1.0.7</a></li>
<li><a href=" https://github.com/servo/rust-url ">form_urlencoded 1.2.1</a></li>
<li><a href=" https://github.com/danburkert/fs2-rs ">fs2 0.4.3</a></li>
@@ -7967,7 +7970,7 @@ limitations under the License.</pre>
<li><a href=" https://github.com/servo/rust-url/ ">idna 0.3.0</a></li>
<li><a href=" https://github.com/servo/rust-url/ ">idna 0.5.0</a></li>
<li><a href=" https://github.com/bluss/indexmap ">indexmap 1.9.3</a></li>
- <li><a href=" https://github.com/indexmap-rs/indexmap ">indexmap 2.2.6</a></li>
+ <li><a href=" https://github.com/indexmap-rs/indexmap ">indexmap 2.3.0</a></li>
<li><a href=" https://github.com/mitsuhiko/insta ">insta 1.39.0</a></li>
<li><a href=" https://github.com/rust-itertools/itertools ">itertools 0.10.5</a></li>
<li><a href=" https://github.com/rust-itertools/itertools ">itertools 0.11.0</a></li>
@@ -7997,7 +8000,7 @@ limitations under the License.</pre>
<li><a href=" https://github.com/rust-num/num-rational ">num-rational 0.4.2</a></li>
<li><a href=" https://github.com/rust-num/num-traits ">num-traits 0.2.19</a></li>
<li><a href=" https://github.com/seanmonstar/num_cpus ">num_cpus 1.16.0</a></li>
- <li><a href=" https://github.com/gimli-rs/object ">object 0.36.2</a></li>
+ <li><a href=" https://github.com/gimli-rs/object ">object 0.36.3</a></li>
<li><a href=" https://github.com/matklad/once_cell ">once_cell 1.19.0</a></li>
<li><a href=" https://github.com/alexcrichton/openssl-probe ">openssl-probe 0.1.5</a></li>
<li><a href=" https://github.com/oxigraph/oxiri ">oxiri 0.2.3</a></li>
@@ -8019,13 +8022,13 @@ limitations under the License.</pre>
<li><a href=" https://github.com/rust-lang/regex ">regex-syntax 0.6.29</a></li>
<li><a href=" https://github.com/rust-lang/regex/tree/master/regex-syntax ">regex-syntax 0.7.5</a></li>
<li><a href=" https://github.com/rust-lang/regex/tree/master/regex-syntax ">regex-syntax 0.8.4</a></li>
- <li><a href=" https://github.com/rust-lang/regex ">regex 1.10.5</a></li>
+ <li><a href=" https://github.com/rust-lang/regex ">regex 1.10.6</a></li>
<li><a href=" https://github.com/rust-lang/rustc-demangle ">rustc-demangle 0.1.24</a></li>
<li><a href=" https://github.com/rust-lang-nursery/rustc-hash ">rustc-hash 1.1.0</a></li>
<li><a href=" https://github.com/Kimundi/rustc-version-rs ">rustc_version 0.4.0</a></li>
<li><a href=" https://github.com/bytecodealliance/rustix ">rustix 0.38.34</a></li>
<li><a href=" https://github.com/rustls/pemfile ">rustls-pemfile 1.0.4</a></li>
- <li><a href=" https://github.com/rustls/pemfile ">rustls-pemfile 2.1.2</a></li>
+ <li><a href=" https://github.com/rustls/pemfile ">rustls-pemfile 2.1.3</a></li>
<li><a href=" https://github.com/rustls/rustls ">rustls 0.23.12</a></li>
<li><a href=" https://github.com/altsysrq/rusty-fork ">rusty-fork 0.3.0</a></li>
<li><a href=" https://github.com/bluss/scopeguard ">scopeguard 1.2.0</a></li>
@@ -8042,7 +8045,7 @@ limitations under the License.</pre>
<li><a href=" https://github.com/dtolnay/syn ">syn 1.0.109</a></li>
<li><a href=" https://github.com/gdesmott/system-deps ">system-deps 6.2.2</a></li>
<li><a href=" https://github.com/alexcrichton/tar-rs ">tar 0.4.41</a></li>
- <li><a href=" https://github.com/Stebalien/tempfile ">tempfile 3.10.1</a></li>
+ <li><a href=" https://github.com/Stebalien/tempfile ">tempfile 3.12.0</a></li>
<li><a href=" https://github.com/servo/tendril ">tendril 0.4.3</a></li>
<li><a href=" https://github.com/Stebalien/term ">term 0.7.0</a></li>
<li><a href=" https://github.com/Amanieu/thread_local-rs ">thread_local 1.1.8</a></li>
@@ -8059,7 +8062,7 @@ limitations under the License.</pre>
<li><a href=" https://github.com/servo/rust-url ">url 2.5.2</a></li>
<li><a href=" https://github.com/uuid-rs/uuid ">uuid 1.10.0</a></li>
<li><a href=" https://github.com/drmingdrmer/validit ">validit 0.2.4</a></li>
- <li><a href=" https://github.com/SergioBenitez/version_check ">version_check 0.9.4</a></li>
+ <li><a href=" https://github.com/SergioBenitez/version_check ">version_check 0.9.5</a></li>
<li><a href=" https://github.com/alexcrichton/wait-timeout ">wait-timeout 0.2.0</a></li>
<li><a href=" https://github.com/bytecodealliance/wasi ">wasi 0.11.0+wasi-snapshot-preview1</a></li>
<li><a href=" https://github.com/rustwasm/wasm-bindgen/tree/master/crates/backend ">wasm-bindgen-backend 0.2.92</a></li>
@@ -10659,7 +10662,7 @@ limitations under the License.
<h3 id="Apache-2.0">Apache License 2.0</h3>
<h4>Used by:</h4>
<ul class="license-used-by">
- <li><a href=" https://github.com/Lokathor/bytemuck ">bytemuck 1.16.1</a></li>
+ <li><a href=" https://github.com/Lokathor/bytemuck ">bytemuck 1.16.3</a></li>
<li><a href=" https://github.com/Lokathor/bytemuck ">bytemuck_derive 1.7.0</a></li>
</ul>
<pre class="license-text">Apache License
@@ -11150,7 +11153,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
<h3 id="BSD-3-Clause">BSD 3-Clause &quot;New&quot; or &quot;Revised&quot; License</h3>
<h4>Used by:</h4>
<ul class="license-used-by">
- <li><a href=" https://github.com/kornelski/cavif-rs ">ravif 0.11.8</a></li>
+ <li><a href=" https://github.com/kornelski/cavif-rs ">ravif 0.11.10</a></li>
</ul>
<pre class="license-text">BSD 3-Clause License
@@ -11381,7 +11384,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
<h3 id="BSL-1.0">Boost Software License 1.0</h3>
<h4>Used by:</h4>
<ul class="license-used-by">
- <li><a href=" https://github.com/DoumanAsh/xxhash-rust ">xxhash-rust 0.8.11</a></li>
+ <li><a href=" https://github.com/DoumanAsh/xxhash-rust ">xxhash-rust 0.8.12</a></li>
</ul>
<pre class="license-text">Boost Software License - Version 1.0 - August 17th, 2003
@@ -12344,7 +12347,6 @@ THE SOFTWARE IS PROVIDED &quot;AS IS&quot;, WITHOUT WARRANTY OF ANY KIND, EXPRES
<h3 id="MIT">MIT License</h3>
<h4>Used by:</h4>
<ul class="license-used-by">
- <li><a href=" https://gitlab.redox-os.org/redox-os/syscall ">redox_syscall 0.4.1</a></li>
<li><a href=" https://gitlab.redox-os.org/redox-os/syscall ">redox_syscall 0.5.3</a></li>
</ul>
<pre class="license-text">Copyright (c) 2017 Redox OS Developers
@@ -12436,7 +12438,7 @@ DEALINGS IN THE SOFTWARE.
<h3 id="MIT">MIT License</h3>
<h4>Used by:</h4>
<ul class="license-used-by">
- <li><a href=" https://github.com/tokio-rs/bytes ">bytes 1.6.1</a></li>
+ <li><a href=" https://github.com/tokio-rs/bytes ">bytes 1.7.1</a></li>
</ul>
<pre class="license-text">Copyright (c) 2018 Carl Lerche
@@ -12934,7 +12936,7 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
<h3 id="MIT">MIT License</h3>
<h4>Used by:</h4>
<ul class="license-used-by">
- <li><a href=" https://github.com/hyperium/hyper-util ">hyper-util 0.1.6</a></li>
+ <li><a href=" https://github.com/hyperium/hyper-util ">hyper-util 0.1.7</a></li>
</ul>
<pre class="license-text">Copyright (c) 2023 Sean McArthur
@@ -13117,7 +13119,7 @@ THE SOFTWARE IS PROVIDED &quot;AS IS&quot;, WITHOUT WARRANTY OF ANY KIND, EXPRES
<h3 id="MIT">MIT License</h3>
<h4>Used by:</h4>
<ul class="license-used-by">
- <li><a href=" https://github.com/jeromefroe/lru-rs.git ">lru 0.12.3</a></li>
+ <li><a href=" https://github.com/jeromefroe/lru-rs.git ">lru 0.12.4</a></li>
</ul>
<pre class="license-text">MIT License
@@ -13529,7 +13531,7 @@ SOFTWARE.
<h3 id="MIT">MIT License</h3>
<h4>Used by:</h4>
<ul class="license-used-by">
- <li><a href=" https://github.com/kornelski/rust-rgb ">rgb 0.8.45</a></li>
+ <li><a href=" https://github.com/kornelski/rust-rgb ">rgb 0.8.48</a></li>
</ul>
<pre class="license-text">MIT License
@@ -14043,7 +14045,7 @@ THE SOFTWARE IS PROVIDED &quot;AS IS&quot;, WITHOUT WARRANTY OF ANY KIND, EXPRES
<h3 id="MIT">MIT License</h3>
<h4>Used by:</h4>
<ul class="license-used-by">
- <li><a href=" https://github.com/tokio-rs/tokio ">tokio 1.39.1</a></li>
+ <li><a href=" https://github.com/tokio-rs/tokio ">tokio 1.39.2</a></li>
</ul>
<pre class="license-text">MIT License
@@ -14191,7 +14193,7 @@ DEALINGS IN THE SOFTWARE.
<h4>Used by:</h4>
<ul class="license-used-by">
<li><a href=" https://github.com/winnow-rs/winnow ">winnow 0.5.40</a></li>
- <li><a href=" https://github.com/winnow-rs/winnow ">winnow 0.6.15</a></li>
+ <li><a href=" https://github.com/winnow-rs/winnow ">winnow 0.6.18</a></li>
</ul>
<pre class="license-text">Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
@@ -14542,7 +14544,7 @@ SOFTWARE.
<h4>Used by:</h4>
<ul class="license-used-by">
<li><a href=" https://github.com/BurntSushi/same-file ">same-file 1.0.6</a></li>
- <li><a href=" https://github.com/BurntSushi/winapi-util ">winapi-util 0.1.8</a></li>
+ <li><a href=" https://github.com/BurntSushi/winapi-util ">winapi-util 0.1.9</a></li>
</ul>
<pre class="license-text">The MIT License (MIT)
diff --git a/crates/core/src/inverted_index/key_phrase.rs b/crates/core/src/inverted_index/key_phrase.rs
index 2726ab3a..853bf4f9 100644
--- a/crates/core/src/inverted_index/key_phrase.rs
+++ b/crates/core/src/inverted_index/key_phrase.rs
@@ -14,120 +14,18 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use itertools::Itertools;
use lending_iter::LendingIterator;
+use std::cmp::Reverse;
+use std::collections::BinaryHeap;
use std::str;
-use std::{cmp::Reverse, collections::HashMap};
use crate::{
schema::text_field::{self, TextField},
SortableFloat,
};
-const DONT_SCORE_TOP_PERCENT_OF_WORDS: f64 = 0.002;
const NON_ALPHABETIC_CHAR_THRESHOLD: f64 = 0.25;
-struct Scorer {
- word_freq: HashMap<String, u64>,
- word_docs: HashMap<String, u64>,
- num_docs: u64,
- word_freq_threshold: u64,
-}
-
-impl Scorer {
- fn new(searcher: &tantivy::Searcher, field: tantivy::schema::Field) -> Self {
- let mut word_freq = HashMap::new();
- let mut word_docs = HashMap::new();
- let mut num_docs = 0;
-
- for seg_reader in searcher.segment_readers() {
- let inv_index = seg_reader.inverted_index(field).unwrap();
- let mut stream = inv_index.terms().stream().unwrap();
-
- while let Some((term, _)) = stream.next() {
- let term_str = str::from_utf8(term).unwrap().to_string();
-
- let words = term_str.split_whitespace().collect::<Vec<_>>();
- if words.is_empty() {
- continue;
- }
-
- for word in &words {
- *word_freq.entry(word.to_string()).or_insert(0) += 1;
- }
-
- for word in words.into_iter().unique() {
- *word_docs.entry(word.to_string()).or_insert(0) += 1;
- }
- }
-
- num_docs += inv_index.terms().num_terms() as u64;
- }
-
- let num_words = word_freq.len() as f64;
- let word_freq_threshold = *word_freq
- .values()
- .sorted_by(|a, b| b.cmp(a))
- .nth((num_words * DONT_SCORE_TOP_PERCENT_OF_WORDS).ceil() as usize)
- .unwrap_or(&0);
-
- Self {
- word_freq,
- word_docs,
- num_docs,
- word_freq_threshold,
- }
- }
-
- #[inline]
- fn word_freq(&self) -> &HashMap<String, u64> {
- &self.word_freq
- }
-
- #[inline]
- fn word_docs(&self) -> &HashMap<String, u64> {
- &self.word_docs
- }
-
- #[inline]
- fn num_docs(&self) -> u64 {
- self.num_docs
- }
-
- #[inline]
- fn word_freq_threshold(&self) -> u64 {
- self.word_freq_threshold
- }
-
- fn score(&self, words: &[&str]) -> f64 {
- let word_freq_threshold = self.word_freq_threshold();
- let mut score = 0.0;
- let num_words = words.len();
- for word in words.iter().unique() {
- let word_chars = word.chars().count();
- if word.chars().filter(|c| !c.is_alphabetic()).count() as f64 / word_chars as f64
- > NON_ALPHABETIC_CHAR_THRESHOLD
- {
- continue;
- }
-
- let word_docs = self.word_docs().get(*word).unwrap_or(&0);
- let wf = *self.word_freq().get(*word).unwrap_or(&0);
-
- if wf > word_freq_threshold {
- continue;
- }
-
- let tf = (wf as f64) / num_words as f64;
- let idf = ((self.num_docs() as f64) / (*word_docs as f64) + 1.0).ln();
-
- score += tf * idf;
- }
-
- score
- }
-}
-
#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
pub struct KeyPhrase {
phrase: String,
@@ -146,14 +44,13 @@ impl KeyPhrase {
.get_field(text_field::KeyPhrases.name())
.unwrap();
- let scorer = Scorer::new(&searcher, field);
-
- let mut keywords: HashMap<String, f64> = HashMap::new();
+ let mut keywords: BinaryHeap<(Reverse<SortableFloat>, String)> =
+ BinaryHeap::with_capacity(top_n);
for seg_reader in searcher.segment_readers() {
let inv_index = seg_reader.inverted_index(field).unwrap();
let mut stream = inv_index.terms().stream().unwrap();
- while let Some((term, _)) = stream.next() {
+ while let Some((term, info)) = stream.next() {
let term_str = str::from_utf8(term).unwrap().to_string();
let num_chars = term_str.chars().count();
@@ -176,24 +73,28 @@ impl KeyPhrase {
continue;
}
- let score = words.len().min(5) as f64 * scorer.score(&words);
+ let score = info.doc_freq as f64;
if score.is_normal() {
let term_str = words.join(" ");
- *keywords.entry(term_str).or_default() += score;
+
+ if keywords.len() >= top_n {
+ if let Some(mut min) = keywords.peek_mut() {
+ if score > min.0 .0.into() {
+ *min = (Reverse(score.into()), term_str);
+ }
+ }
+ } else {
+ keywords.push((Reverse(score.into()), term_str));
+ }
}
}
}
- crate::sorted_k(
- keywords
- .into_iter()
- .map(|(phrase, score)| (Reverse(SortableFloat::from(score)), phrase)),
- top_n,
- )
- .into_iter()
- .map(|(Reverse(score), phrase)| KeyPhrase::new(phrase, score.into()))
- .collect()
+ keywords
+ .into_iter()
+ .map(|(Reverse(score), phrase)| KeyPhrase::new(phrase, score.into()))
+ .collect()
}
pub fn score(&self) -> f64 {