web search done right
Ftr/spellcheck (#14)
Mikkel Denkerc5ec4f46f15c
diff --git a/Cargo.lock b/Cargo.lockindex b11a824e..2ac2347d 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -582,6 +582,7 @@ dependencies = ["lru","maplit","md5",+ "memmap","nom","parse_wiki_text","quick-xml",@@ -797,6 +798,9 @@ name = "fst"version = "0.4.7"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "7ab85b9b05e3978cc9a9cf8fea7f01b494e1a09ed3037e16ba39edc7a29eb61a"+dependencies = [+ "utf8-ranges",+][[package]]name = "futures"@@ -1444,6 +1448,16 @@ version = "2.5.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"+[[package]]+name = "memmap"+version = "0.7.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b"+dependencies = [+ "libc",+ "winapi",+]+[[package]]name = "memmap2"version = "0.5.4"diff --git a/Cargo.toml b/Cargo.tomlindex 666e6e88..ae349940 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -41,7 +41,7 @@ axum-extra = { version = "0.3.4", features = ["spa"] }html-escape = "0.2.11"logos = "0.12.1"csv = "1.1.6"-fst = "0.4.7"+fst = { version = "0.4.7", features = ["levenshtein"] }lazy_static = "1.4.0"rocksdb = "0.18.0"image = "0.24.3"@@ -52,6 +52,7 @@ quick-xml = "0.23.0"bzip2 = "0.4.3"parse_wiki_text = "0.1.5"md5 = "0.7.0"+memmap = "0.7.0"[dev-dependencies]maplit = "1.0.2"diff --git a/data/entity/attribute_occurrences/OPTIONS-000135 b/data/entity/attribute_occurrences/OPTIONS-000135deleted file mode 100644index 9f62adb9..00000000--- a/data/entity/attribute_occurrences/OPTIONS-000135+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:9d110b3864bbce0573ace88015c3b8da37878518c24f398ba98390ffe012efdc-size 6535diff --git a/data/entity/images/OPTIONS-000147 b/data/entity/images/OPTIONS-000147deleted file mode 100644index 9f62adb9..00000000--- a/data/entity/images/OPTIONS-000147+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:9d110b3864bbce0573ace88015c3b8da37878518c24f398ba98390ffe012efdc-size 6535diff --git a/data/index/favicon_store/000009.sst b/data/index/favicon_store/000009.sstindex e35b63eb..62f314de 100644--- a/data/index/favicon_store/000009.sst+++ b/data/index/favicon_store/000009.sst@@ -1,3 +1,3 @@version https://git-lfs.github.com/spec/v1-oid sha256:8ddc6d4940e8aef7aceb41a85e47e10919bc2ceeb424b2e0e461549ca5d44d61-size 89167+oid sha256:fa3cad28e7129c270895f1f2376ce5e635036e7d3b632106d5109bc59f106be2+size 133796diff --git a/data/index/favicon_store/IDENTITY b/data/index/favicon_store/IDENTITYindex c5717c62..0eea3b44 100644--- a/data/index/favicon_store/IDENTITY+++ b/data/index/favicon_store/IDENTITY@@ -1,3 +1,3 @@version https://git-lfs.github.com/spec/v1-oid sha256:740eb887bf29fb004308c69e067b676c0f54ace10fb5f678c0ea08a2496d95ba+oid sha256:d13bef3c0e708ed912ebf554c23fdb705667714a71092dffd659d562875b9729size 36diff --git a/data/index/favicon_store/LOG.old.1660643834782411 b/data/index/favicon_store/LOG.old.1660643834782411deleted file mode 100644index 10b796ba..00000000--- a/data/index/favicon_store/LOG.old.1660643834782411+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:b5c482c1291fb84b3d412a0036dcdfbbed93aa5255236135b4a074c4ba5bbcf5-size 26146diff --git a/data/index/favicon_store/LOG.old.1660643841806793 b/data/index/favicon_store/LOG.old.1660643841806793deleted file mode 100644index fcbcaa0a..00000000--- a/data/index/favicon_store/LOG.old.1660643841806793+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:fc85b5da8d22a07cff04ee71d6f2248c6b1151ea61e6aec34e7b238bd3f25473-size 20225diff --git a/data/index/favicon_store/LOG.old.1660643992979668 b/data/index/favicon_store/LOG.old.1660643992979668deleted file mode 100644index 4d8833a2..00000000--- a/data/index/favicon_store/LOG.old.1660643992979668+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:357c4622739edd1087d52615a0dec7196f6e48b0d7d92af9c0924a1f324dfb68-size 22900diff --git a/data/index/favicon_store/LOG.old.1660644125900052 b/data/index/favicon_store/LOG.old.1660644125900052deleted file mode 100644index c356d66c..00000000--- a/data/index/favicon_store/LOG.old.1660644125900052+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:64f1021d417aa828cdcc6c45651bb41e557f13c444be1aa1bd8b66a8a9d2a6d3-size 22902diff --git a/data/index/favicon_store/LOG.old.1660644236142610 b/data/index/favicon_store/LOG.old.1660644236142610deleted file mode 100644index b9a7b2ac..00000000--- a/data/index/favicon_store/LOG.old.1660644236142610+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:3c7b70ac6206396332085efc106fcc046cec6e8e8fbe0223604969c62c863b3c-size 22935diff --git a/data/index/favicon_store/LOG.old.1660644357270366 b/data/index/favicon_store/LOG.old.1660644357270366deleted file mode 100644index 43c036b1..00000000--- a/data/index/favicon_store/LOG.old.1660644357270366+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:3a4e1ae4f35e359c1f5ea4ffc11ab64aed7df14a69d2b4bcad3106023ac1cab7-size 22934diff --git a/data/index/favicon_store/LOG.old.1660649840289268 b/data/index/favicon_store/LOG.old.1660649840289268deleted file mode 100644index 477666cb..00000000--- a/data/index/favicon_store/LOG.old.1660649840289268+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:42ab28c8deaa48ffbf17f896868db0c80a34981c20560869065b4fef9e970aad-size 22934diff --git a/data/index/favicon_store/LOG.old.1660649960425303 b/data/index/favicon_store/LOG.old.1660649960425303deleted file mode 100644index 46ca663b..00000000--- a/data/index/favicon_store/LOG.old.1660649960425303+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:2694cdc5c84a20196b53e991ae2ee5c27553a61706def5c57efba32cd1268b99-size 22935diff --git a/data/index/favicon_store/LOG.old.1660650026549335 b/data/index/favicon_store/LOG.old.1660650026549335deleted file mode 100644index a6342ccc..00000000--- a/data/index/favicon_store/LOG.old.1660650026549335+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:ba6daeaa11629123e9bd7b16b9f6e1218d6e1eb2178dde94ed2d1a9c6c974e73-size 22934diff --git a/data/index/favicon_store/LOG.old.1660650103901274 b/data/index/favicon_store/LOG.old.1660650103901274deleted file mode 100644index f6c3d3ed..00000000--- a/data/index/favicon_store/LOG.old.1660650103901274+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:46bc5f5de71273bc90a677d403e1cc0abf5056f7c74d2794a56022e6a25c269c-size 22932diff --git a/data/index/favicon_store/LOG.old.1660651249991160 b/data/index/favicon_store/LOG.old.1660651249991160deleted file mode 100644index 84b321b4..00000000--- a/data/index/favicon_store/LOG.old.1660651249991160+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:bc049602a3cafe817c6e1b0bf75a9021ffbf3a1b6ae372f541270cf85bda40f5-size 22933diff --git a/data/index/favicon_store/OPTIONS-000161 b/data/index/favicon_store/OPTIONS-000161deleted file mode 100644index 9f62adb9..00000000--- a/data/index/favicon_store/OPTIONS-000161+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:9d110b3864bbce0573ace88015c3b8da37878518c24f398ba98390ffe012efdc-size 6535diff --git a/data/index/inverted_index/.managed.json b/data/index/inverted_index/.managed.jsonindex e9787a55..1ba9113d 100644--- a/data/index/inverted_index/.managed.json+++ b/data/index/inverted_index/.managed.json@@ -1,3 +1,3 @@version https://git-lfs.github.com/spec/v1-oid sha256:7e3a2db879d6be2cd18aa86e23e3dd9717c565ca4383b396de978d50c11f27c9+oid sha256:6fc73ac5556fab2417af3b089d9e51b0475ba3308f99153982fd3eb2c63f9d1esize 258diff --git a/data/index/inverted_index/381ccf14f7454ae6adf96a643f363ce8.fast b/data/index/inverted_index/381ccf14f7454ae6adf96a643f363ce8.fastdeleted file mode 100644index ab585503..00000000--- a/data/index/inverted_index/381ccf14f7454ae6adf96a643f363ce8.fast+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:f7bff4b7235b7d1e87d365551963e4a24756bbaef5abbfb1ce4d274a19377610-size 222787diff --git a/data/index/inverted_index/381ccf14f7454ae6adf96a643f363ce8.fieldnorm b/data/index/inverted_index/381ccf14f7454ae6adf96a643f363ce8.fieldnormdeleted file mode 100644index d94c7d3e..00000000--- a/data/index/inverted_index/381ccf14f7454ae6adf96a643f363ce8.fieldnorm+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:8ce278866b99fcd5bbe712a48185107b1d7d330d6b1185ecec2f3cee83ddd483-size 318674diff --git a/data/index/inverted_index/381ccf14f7454ae6adf96a643f363ce8.idx b/data/index/inverted_index/381ccf14f7454ae6adf96a643f363ce8.idxdeleted file mode 100644index cf9e9074..00000000--- a/data/index/inverted_index/381ccf14f7454ae6adf96a643f363ce8.idx+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:82045379ae39b536f09e46b22987f4b6991d26c2066649994253dab50792115b-size 54219587diff --git a/data/index/inverted_index/381ccf14f7454ae6adf96a643f363ce8.pos b/data/index/inverted_index/381ccf14f7454ae6adf96a643f363ce8.posdeleted file mode 100644index 24f1180b..00000000--- a/data/index/inverted_index/381ccf14f7454ae6adf96a643f363ce8.pos+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:aa5f3acd93c379b463ce4e732d5c9302f377cc890bd201227a8ea71d4dca1e7d-size 111367468diff --git a/data/index/inverted_index/381ccf14f7454ae6adf96a643f363ce8.store b/data/index/inverted_index/381ccf14f7454ae6adf96a643f363ce8.storedeleted file mode 100644index c2777e07..00000000--- a/data/index/inverted_index/381ccf14f7454ae6adf96a643f363ce8.store+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:511e6fb2d03a58bc14408bc2188192953ac69561b6a84dca18a1279ed32741b7-size 108014845diff --git a/data/index/inverted_index/381ccf14f7454ae6adf96a643f363ce8.term b/data/index/inverted_index/381ccf14f7454ae6adf96a643f363ce8.termdeleted file mode 100644index f8f7402f..00000000--- a/data/index/inverted_index/381ccf14f7454ae6adf96a643f363ce8.term+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:a99487b51229045bde17902e2ea3c12e1c2e4d3b0ca88257425c6b44c099facd-size 85274891diff --git a/data/index/inverted_index/8842c700af704f7db01221862b8fee35.fast b/data/index/inverted_index/8842c700af704f7db01221862b8fee35.fastnew file mode 100644index 00000000..74692838--- /dev/null+++ b/data/index/inverted_index/8842c700af704f7db01221862b8fee35.fast@@ -0,0 +1,3 @@+version https://git-lfs.github.com/spec/v1+oid sha256:55c9745319da0509b16c0cc0d2170b76168ff88b24e1cf64b80073d578734a05+size 224771diff --git a/data/index/inverted_index/8842c700af704f7db01221862b8fee35.fieldnorm b/data/index/inverted_index/8842c700af704f7db01221862b8fee35.fieldnormnew file mode 100644index 00000000..e97b8610--- /dev/null+++ b/data/index/inverted_index/8842c700af704f7db01221862b8fee35.fieldnorm@@ -0,0 +1,3 @@+version https://git-lfs.github.com/spec/v1+oid sha256:dfd123ad020d5f2d56a5ee465115aba35ada8d44e26407a37e8da69a2ad171df+size 318675diff --git a/data/index/inverted_index/8842c700af704f7db01221862b8fee35.idx b/data/index/inverted_index/8842c700af704f7db01221862b8fee35.idxnew file mode 100644index 00000000..62b2e139--- /dev/null+++ b/data/index/inverted_index/8842c700af704f7db01221862b8fee35.idx@@ -0,0 +1,3 @@+version https://git-lfs.github.com/spec/v1+oid sha256:6151ebd74b0b9f557d6ec30b521aa3c980931c722b2ac93b55280045a162828f+size 54160567diff --git a/data/index/inverted_index/8842c700af704f7db01221862b8fee35.pos b/data/index/inverted_index/8842c700af704f7db01221862b8fee35.posnew file mode 100644index 00000000..568f2c51--- /dev/null+++ b/data/index/inverted_index/8842c700af704f7db01221862b8fee35.pos@@ -0,0 +1,3 @@+version https://git-lfs.github.com/spec/v1+oid sha256:1be7c947556c915dc2eef0a3b78e4767e8b70c8a28b7c940eba157e895c8b1ed+size 111365445diff --git a/data/index/inverted_index/8842c700af704f7db01221862b8fee35.store b/data/index/inverted_index/8842c700af704f7db01221862b8fee35.storenew file mode 100644index 00000000..0765e6fe--- /dev/null+++ b/data/index/inverted_index/8842c700af704f7db01221862b8fee35.store@@ -0,0 +1,3 @@+version https://git-lfs.github.com/spec/v1+oid sha256:0c9ae983092355587f2993ebefe9bc8c03b8d72e105bdf5d15b579b89cca48f9+size 108022084diff --git a/data/index/inverted_index/8842c700af704f7db01221862b8fee35.term b/data/index/inverted_index/8842c700af704f7db01221862b8fee35.termnew file mode 100644index 00000000..0066651f--- /dev/null+++ b/data/index/inverted_index/8842c700af704f7db01221862b8fee35.term@@ -0,0 +1,3 @@+version https://git-lfs.github.com/spec/v1+oid sha256:28842f363d110638b3673d229576ab0488fcb9c0819af87bf12dc3136a722a46+size 85271351diff --git a/data/index/inverted_index/meta.json b/data/index/inverted_index/meta.jsonindex f2841b5a..cce3f738 100644--- a/data/index/inverted_index/meta.json+++ b/data/index/inverted_index/meta.json@@ -1,3 +1,3 @@version https://git-lfs.github.com/spec/v1-oid sha256:715b50d87576bfb1794a476727b33f2377810df5b7e5a4719951c814940548a1+oid sha256:3c2c0ad7e61da59eddc5a42053bca7f84e85ae8f600cde7b10046c2f3c7d7e54size 4541diff --git a/data/index/primary_image_store/000009.sst b/data/index/primary_image_store/000009.sstindex dacaa1f5..eefd8bd9 100644--- a/data/index/primary_image_store/000009.sst+++ b/data/index/primary_image_store/000009.sst@@ -1,3 +1,3 @@version https://git-lfs.github.com/spec/v1-oid sha256:0a277ac634dbb9632f40f8e05abf5264bb621238d8a93ac6117888449f4bba36-size 26218869+oid sha256:a846b78278d422997fd1e9cc7849f10d2bdf4d3dc320436f8064d0c78921d778+size 26538187diff --git a/data/index/primary_image_store/IDENTITY b/data/index/primary_image_store/IDENTITYindex 51bbb310..4095860a 100644--- a/data/index/primary_image_store/IDENTITY+++ b/data/index/primary_image_store/IDENTITY@@ -1,3 +1,3 @@version https://git-lfs.github.com/spec/v1-oid sha256:ea55c46eebd189a9be90f18ef25a3caea033fd2c5b82ee2eb26adc57bd2bdd80+oid sha256:766cac6c9a3d17b8a1d67c1b5da99318619dfa21c3aca032c01fcdb3d4547463size 36diff --git a/data/index/primary_image_store/LOG.old.1660643834784950 b/data/index/primary_image_store/LOG.old.1660643834784950deleted file mode 100644index 78329b7b..00000000--- a/data/index/primary_image_store/LOG.old.1660643834784950+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:c055c05cbfd1b9915334d940e0c0a4113c800d1583a61de40fef492db0e8a322-size 26213diff --git a/data/index/primary_image_store/LOG.old.1660643841811016 b/data/index/primary_image_store/LOG.old.1660643841811016deleted file mode 100644index e34353a7..00000000--- a/data/index/primary_image_store/LOG.old.1660643841811016+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:d7e99db95ec0481e6d9e6d19ce6f92577e70c81a301fd7b80914f56c7db9611d-size 20257diff --git a/data/index/primary_image_store/LOG.old.1660643992984221 b/data/index/primary_image_store/LOG.old.1660643992984221deleted file mode 100644index 7b5b4ebd..00000000--- a/data/index/primary_image_store/LOG.old.1660643992984221+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:115073258401b92e1aacfb6ae188e078355bedb0a41c8bccd1d8234b5c56f8aa-size 22931diff --git a/data/index/primary_image_store/LOG.old.1660644125904383 b/data/index/primary_image_store/LOG.old.1660644125904383deleted file mode 100644index f224e3cd..00000000--- a/data/index/primary_image_store/LOG.old.1660644125904383+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:8c4dc0ced0085e73914b06749b60e05cdb67c8d116f726cf35d88172a07e56f2-size 22934diff --git a/data/index/primary_image_store/LOG.old.1660644236146938 b/data/index/primary_image_store/LOG.old.1660644236146938deleted file mode 100644index aa1b14b3..00000000--- a/data/index/primary_image_store/LOG.old.1660644236146938+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:f37a2cc4a9e0b9218fb5f4c4be27e5dd630fd00eddbaf3c263bba94cb36fe553-size 22967diff --git a/data/index/primary_image_store/LOG.old.1660644357272670 b/data/index/primary_image_store/LOG.old.1660644357272670deleted file mode 100644index d17f7cf9..00000000--- a/data/index/primary_image_store/LOG.old.1660644357272670+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:1bff841f7bf97e4b3f3b6a9234aa826257012cb758f9a49e59fa5c4cd436397f-size 22966diff --git a/data/index/primary_image_store/LOG.old.1660649840294017 b/data/index/primary_image_store/LOG.old.1660649840294017deleted file mode 100644index 6d4ce341..00000000--- a/data/index/primary_image_store/LOG.old.1660649840294017+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:51f43612649838caba68b88842d858da69c6f700febd0194b1ad260cc2791650-size 22963diff --git a/data/index/primary_image_store/LOG.old.1660649960427655 b/data/index/primary_image_store/LOG.old.1660649960427655deleted file mode 100644index 0fc76d3f..00000000--- a/data/index/primary_image_store/LOG.old.1660649960427655+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:86fb7ab2e29c2f8e5103abfbc9711f8e7fcfb672a7a2669538d320258ee33f94-size 22967diff --git a/data/index/primary_image_store/LOG.old.1660650026552591 b/data/index/primary_image_store/LOG.old.1660650026552591deleted file mode 100644index 22163210..00000000--- a/data/index/primary_image_store/LOG.old.1660650026552591+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:cf9c379fd750cbe54aa0e46beaa351c7ea4ad0f21856f1449de52868945f73f5-size 22965diff --git a/data/index/primary_image_store/LOG.old.1660650103903637 b/data/index/primary_image_store/LOG.old.1660650103903637deleted file mode 100644index 5f5a20a2..00000000--- a/data/index/primary_image_store/LOG.old.1660650103903637+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:0056c77eeb1707d7f68ed856929332c7870724dc50bb7a773abdbb32b4bdb127-size 22965diff --git a/data/index/primary_image_store/LOG.old.1660651249995469 b/data/index/primary_image_store/LOG.old.1660651249995469deleted file mode 100644index d847ede1..00000000--- a/data/index/primary_image_store/LOG.old.1660651249995469+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:00d6d4157444dc238cfdef8ee130778ba2c67d5b5c6160ec25b8641a3136b95c-size 22933diff --git a/data/index/primary_image_store/OPTIONS-000161 b/data/index/primary_image_store/OPTIONS-000161deleted file mode 100644index 9f62adb9..00000000--- a/data/index/primary_image_store/OPTIONS-000161+++ /dev/null@@ -1,3 +0,0 @@-version https://git-lfs.github.com/spec/v1-oid sha256:9d110b3864bbce0573ace88015c3b8da37878518c24f398ba98390ffe012efdc-size 6535diff --git a/data/index/primary_image_store/dictionary b/data/index/primary_image_store/dictionarynew file mode 100644index 00000000..2452fa9d--- /dev/null+++ b/data/index/primary_image_store/dictionary@@ -0,0 +1,3 @@+version https://git-lfs.github.com/spec/v1+oid sha256:ffc57eb861fab46f5c16cffbe9a692a1beb072a1282bd325562a3a90928b9367+size 5141310diff --git a/src/frontend/search.rs b/src/frontend/search.rsindex 022d8897..c39d9238 100644--- a/src/frontend/search.rs+++ b/src/frontend/search.rs@@ -234,6 +234,7 @@ struct SearchTemplate {search_result: Vec<DisplayedWebpage>,query: String,entity: Option<DisplayedEntity>,+ spell_correction: Option<String>,}pub async fn route(@@ -243,6 +244,7 @@ pub async fn route(let mut search_result = Vec::new();let mut displayed_query = String::new();let mut entity = None;+ let mut spell_correction = None;if let Some(query) = params.get("q") {displayed_query = query.clone();@@ -269,12 +271,14 @@ pub async fn route(entity = result.entity.map(|entity| DisplayedEntity::from(entity, &state.searcher));+ spell_correction = result.spell_corrected_query;}let template = SearchTemplate {search_result,query: displayed_query,entity,+ spell_correction,};HtmlTemplate(template)}diff --git a/src/index.rs b/src/index.rsindex ea9b7f42..ecc9bcae 100644--- a/src/index.rs+++ b/src/index.rs@@ -20,6 +20,7 @@ use std::path::Path;use std::sync::Arc;use std::time::Duration;+use itertools::intersperse;use serde::{Deserialize, Serialize};use tantivy::collector::Collector;use tantivy::schema::Schema;@@ -30,12 +31,14 @@ use crate::image_downloader::{ImageDownloadJob, ImageDownloader};use crate::image_store::{FaviconStore, Image, ImageStore, PrimaryImageStore};use crate::inverted_index::{InvertedIndex, InvertedIndexSearchResult};use crate::query::Query;+use crate::spell::{Dictionary, LogarithmicEdit, SpellChecker, TermSplitter};use crate::webpage::{Url, Webpage};use crate::Result;const INVERTED_INDEX_SUBFOLDER_NAME: &str = "inverted_index";const FAVICON_STORE_SUBFOLDER_NAME: &str = "favicon_store";const PRIMARY_IMAGE_STORE_SUBFOLDER_NAME: &str = "primary_image_store";+const SPELL_SUBFOLDER_NAME: &str = "primary_image_store";const IMAGE_WEBPAGE_CENTRALITY_THRESHOLD: f64 = 0.0;pub struct Index {@@ -44,6 +47,7 @@ pub struct Index {primary_image_store: PrimaryImageStore,favicon_downloader: ImageDownloader<String>,primary_image_downloader: ImageDownloader<Uuid>,+ spell_dictionary: Dictionary<1_000_000>,pub path: String,}@@ -66,6 +70,7 @@ impl Index {primary_image_store,primary_image_downloader: ImageDownloader::new(),favicon_downloader: ImageDownloader::new(),+ spell_dictionary: Dictionary::open(Some(path.as_ref().join(SPELL_SUBFOLDER_NAME)))?,path: path.as_ref().to_str().unwrap().to_string(),})}@@ -79,11 +84,13 @@ impl Index {pub fn insert(&mut self, mut webpage: Webpage) -> Result<()> {self.maybe_insert_favicon(&webpage);self.maybe_insert_primary_image(&mut webpage);+ self.spell_dictionary.insert_page(&webpage);self.inverted_index.insert(webpage)}pub fn commit(&mut self) -> Result<()> {self.inverted_index.merge_all_segments()?;+ self.spell_dictionary.commit()?;self.inverted_index.commit()}@@ -103,6 +110,8 @@ impl Index {self.primary_image_store.merge(other.primary_image_store);drop(self.primary_image_store);+ self.spell_dictionary.merge(other.spell_dictionary);+Self::open(&self.path).expect("failed to open index")}@@ -162,6 +171,61 @@ impl Index {self.primary_image_downloader.download(&mut self.primary_image_store);}++ fn spell_check(&self, terms: &[String]) -> Option<String> {+ let spellchecker = SpellChecker::new(&self.spell_dictionary, LogarithmicEdit::new(4));+ let mut corrections: Vec<String> = Vec::new();++ for term in terms {+ match spellchecker.correct(term.to_ascii_lowercase().as_str()) {+ Some(correction) => corrections.push(correction.to_ascii_lowercase()),+ None => corrections.push(term.to_ascii_lowercase()),+ }+ }++ if corrections+ .iter()+ .cloned()+ .zip(terms.iter().map(|term| term.to_ascii_lowercase()))+ .all(|(correction, term)| correction == term)+ {+ None+ } else {+ Some(intersperse(corrections.into_iter(), " ".to_string()).collect())+ }+ }++ fn split_words(&self, terms: &[String]) -> Option<String> {+ let splitter = TermSplitter::new(&self.spell_dictionary);+ let mut corrections: Vec<String> = Vec::new();++ for term in terms {+ let t = term.to_ascii_lowercase();+ let split = splitter.split(t.as_str());+ if split.is_empty() {+ corrections.push(term.clone());+ } else {+ for s in split {+ corrections.push(s.to_string())+ }+ }+ }++ if corrections+ .iter()+ .cloned()+ .zip(terms.iter().map(|term| term.to_ascii_lowercase()))+ .all(|(correction, term)| correction == term)+ {+ None+ } else {+ Some(intersperse(corrections.into_iter(), " ".to_string()).collect())+ }+ }++ pub fn spell_correction(&self, terms: &[String]) -> Option<String> {+ self.spell_check(terms).or_else(|| self.split_words(terms))+ }}#[derive(Debug, Serialize, Deserialize)]@@ -234,6 +298,8 @@ mod tests {)).expect("failed to parse webpage");+ index.commit().unwrap();+let path = index.path.clone();let frozen: FrozenIndex = index.into();let bytes = bincode::serialize(&frozen).unwrap();@@ -251,5 +317,47 @@ mod tests {assert_eq!(result.num_docs, 1);assert_eq!(result.documents.len(), 1);assert_eq!(result.documents[0].url, "https://www.example.com");++ assert_eq!(+ index.spell_correction(&["thiss".to_string()]),+ Some("this".to_string())+ );+ }++ #[test]+ fn sentence_spell_correction() {+ let mut index = Index::temporary().expect("Unable to open index");++ index+ .insert(Webpage::new(+ &format!(+ r#"+ <html>+ <head>+ <title>Test website</title>+ </head>+ <body>+ {CONTENT}+ </body>+ </html>+ "#+ ),+ "https://www.example.com",+ vec![],+ 1.0,+ 500,+ ))+ .expect("failed to parse webpage");++ index.commit().unwrap();++ assert_eq!(+ index.spell_correction(&["th".to_string(), "best".to_string()]),+ Some("the best".to_string())+ );+ assert_eq!(+ index.spell_correction(&["the".to_string(), "best".to_string()]),+ None+ );}}diff --git a/src/lib.rs b/src/lib.rsindex 107ea854..97838062 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -13,6 +13,9 @@//// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.++// #![warn(clippy::pedantic)]+use serde::{Deserialize, Serialize};use std::fs::File;use std::io::{self, BufRead};@@ -41,6 +44,7 @@ mod schema;mod schema_org;pub mod searcher;mod snippet;+mod spell;mod tokenizer;mod warc;pub mod webgraph;@@ -178,6 +182,9 @@ pub enum Error {#[error("XML parser error")]XML(#[from] quick_xml::Error),++ #[error("Spell dictionary error")]+ Spell(#[from] crate::spell::dictionary::DictionaryError),}pub(crate) type Result<T> = std::result::Result<T, Error>;diff --git a/src/query/mod.rs b/src/query/mod.rsindex 73d4d9b2..34ea52df 100644--- a/src/query/mod.rs+++ b/src/query/mod.rs@@ -92,6 +92,10 @@ impl Query {Err(error) => Err(Error::ParsingError(error.to_string())),}}++ pub fn simple_terms(&self) -> &[String] {+ &self.terms+ }}impl tantivy::query::Query for Query {diff --git a/src/searcher.rs b/src/searcher.rsindex 8e43e832..8b1d11d3 100644--- a/src/searcher.rs+++ b/src/searcher.rs@@ -29,6 +29,7 @@ use crate::Result;#[derive(Debug)]pub struct SearchResult {+ pub spell_corrected_query: Option<String>,pub webpages: InvertedIndexSearchResult,pub entity: Option<StoredEntity>,}@@ -59,13 +60,18 @@ impl Searcher {let query = Query::parse(query, self.index.schema())?;let ranker = Ranker::new(query.clone());let webpages = self.index.search(&query, ranker.collector())?;+ let correction = self.index.spell_correction(query.simple_terms());let entity = self.entity_index.as_ref().and_then(|index| index.search(raw_query));- Ok(SearchResult { webpages, entity })+ Ok(SearchResult {+ webpages,+ entity,+ spell_corrected_query: correction,+ })}pub fn favicon(&self, site: &Url) -> Option<Image> {diff --git a/src/spell/dictionary.rs b/src/spell/dictionary.rsnew file mode 100644index 00000000..74660a96--- /dev/null+++ b/src/spell/dictionary.rs@@ -0,0 +1,475 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.+use crate::spell::distance::LevenshteinDistance;+use crate::webpage::Webpage;+use fst::map::Union;+use fst::{Automaton, IntoStreamer, Map, MapBuilder, Streamer};+use memmap::Mmap;+use std::collections::{BTreeMap, BinaryHeap, HashSet};+use std::fs::{self, File, OpenOptions};+use std::hash::{Hash, Hasher};+use std::ops::AddAssign;+use std::path::Path;+use std::{cmp, io, mem};+use thiserror::Error;++pub trait EditStrategy: Send + Sync {+ fn distance_for_string(&self, string: &str) -> usize;+ fn dist(&self) -> LevenshteinDistance;+}++pub struct LogarithmicEdit {+ max_edit_distance: usize,+}++impl LogarithmicEdit {+ pub fn new(max_edit_distance: usize) -> Self {+ Self { max_edit_distance }+ }+}++impl EditStrategy for LogarithmicEdit {+ fn distance_for_string(&self, string: &str) -> usize {+ let log_value: usize = (string.len() as f32).log2() as usize;+ cmp::max(1, cmp::min(log_value, self.max_edit_distance))+ }++ fn dist(&self) -> LevenshteinDistance {+ LevenshteinDistance::new(self.max_edit_distance)+ }+}++#[cfg(test)]+pub struct MaxEdit {+ max_edit_distance: usize,+}++#[cfg(test)]+impl MaxEdit {+ pub fn new(max_edit_distance: usize) -> Self {+ Self { max_edit_distance }+ }+}++#[cfg(test)]+impl EditStrategy for MaxEdit {+ fn distance_for_string(&self, _: &str) -> usize {+ self.max_edit_distance+ }++ fn dist(&self) -> LevenshteinDistance {+ LevenshteinDistance::new(self.max_edit_distance)+ }+}++pub struct DictionaryResult {+ pub prob: f64,+ pub correction: String,+}++impl Hash for DictionaryResult {+ fn hash<H>(&self, state: &mut H)+ where+ H: Hasher,+ {+ self.correction.hash(state);+ }+}++impl PartialEq for DictionaryResult {+ fn eq(&self, other: &Self) -> bool {+ self.correction == other.correction+ }+}++impl Eq for DictionaryResult {}++#[derive(Error, Debug)]+pub enum DictionaryError {+ #[error("Underlying error from FST")]+ Fst(#[from] fst::Error),++ #[error("An IO error has occured")]+ Io(#[from] io::Error),+}++pub type Result<T> = std::result::Result<T, DictionaryError>;++enum InnerMap {+ Memory(Map<Vec<u8>>),+ File(Map<Mmap>),+}++impl InnerMap {+ fn total_freq(&self) -> u64 {+ let mut stream = match self {+ InnerMap::Memory(map) => map.into_stream(),+ InnerMap::File(map) => map.into_stream(),+ };++ let mut total_freq = 0;+ while let Some((_, freq)) = stream.next() {+ total_freq += freq;+ }++ total_freq+ }+}++/// Dictionary that contains term frequency information+pub struct Dictionary<const TOP_N: usize> {+ cache: BTreeMap<String, u64>,+ map: InnerMap,+ folder_path: Option<String>,+ total_freq: u64,+}++#[cfg(test)]+impl Default for Dictionary<1_000> {+ fn default() -> Self {+ Dictionary::open::<&str>(None).unwrap()+ }+}++impl<const TOP_N: usize> Dictionary<TOP_N> {+ pub fn open<P: AsRef<Path>>(folder_path: Option<P>) -> Result<Self> {+ if let Some(path) = folder_path.as_ref() {+ fs::create_dir_all(path)?;+ }++ let folder_path = folder_path.map(|path| path.as_ref().to_str().unwrap().to_string());++ let map = match &folder_path {+ Some(path) => {+ let dictionary_path = Path::new(path).join("dictionary");++ if !dictionary_path.exists() {+ let file = File::create(dictionary_path.clone())?;+ let builder = MapBuilder::new(io::BufWriter::new(file))?;+ builder.finish()?;+ }++ let dictionary_file = OpenOptions::new().read(true).open(dictionary_path)?;++ let mmap = unsafe { Mmap::map(&dictionary_file)? };+ InnerMap::File(Map::new(mmap)?)+ }+ None => InnerMap::Memory(Map::default()),+ };++ Ok(Dictionary {+ total_freq: map.total_freq(),+ map,+ folder_path,+ cache: BTreeMap::default(),+ })+ }++ fn store_union(&mut self, mut union: Union) -> Result<()> {+ match &self.folder_path {+ Some(path) => {+ let path = Path::new(path);+ let wrt = io::BufWriter::new(File::create(path.join("new_dictionary"))?);+ let mut builder = MapBuilder::new(wrt)?;++ let mut heap = BinaryHeap::with_capacity(TOP_N + 1);++ while let Some((key, values)) = union.next() {+ let val: u64 = values.iter().map(|idx_val| idx_val.value).sum();++ heap.push((+ std::cmp::Reverse(val),+ String::from_utf8_lossy(key).to_string(),+ ));++ if heap.len() > TOP_N {+ heap.pop();+ }+ }++ let top_values: BTreeMap<_, _> =+ heap.into_iter().map(|(val, key)| (key, val.0)).collect();++ for (key, value) in top_values {+ builder.insert(key, value)?;+ }++ let path = Path::new(path);+ builder.finish()?;+ std::fs::rename(path.join("new_dictionary"), path.join("dictionary"))?;+ let mmap = unsafe { Mmap::map(&File::open(path.join("dictionary"))?)? };+ self.map = InnerMap::File(Map::new(mmap)?);+ }+ None => {+ let mut builder = MapBuilder::memory();++ let mut heap = BinaryHeap::with_capacity(TOP_N + 1);++ while let Some((key, values)) = union.next() {+ let val: u64 = values.iter().map(|idx_val| idx_val.value).sum();++ heap.push((+ std::cmp::Reverse(val),+ String::from_utf8_lossy(key).to_string(),+ ));++ if heap.len() > TOP_N {+ heap.pop();+ }+ }++ let top_values: BTreeMap<_, _> =+ heap.into_iter().map(|(val, key)| (key, val.0)).collect();++ for (key, value) in top_values {+ builder.insert(key, value)?;+ }++ let bytes = builder.into_inner().unwrap();+ self.map = InnerMap::Memory(Map::new(bytes)?);+ }+ };++ Ok(())+ }++ pub fn commit(&mut self) -> Result<()> {+ let cache = mem::take(&mut self.cache);+ let cache_map = Map::from_iter(cache.into_iter())?;+ let map = mem::replace(&mut self.map, InnerMap::Memory(Map::default()));++ let union = match &map {+ InnerMap::File(map) => map.op().add(&cache_map).union(),+ InnerMap::Memory(map) => map.op().add(&cache_map).union(),+ };+ self.store_union(union)?;++ self.total_freq = self.map.total_freq();++ Ok(())+ }++ pub fn insert(&mut self, term: &str) {+ self.cache+ .entry(+ term.chars()+ .into_iter()+ .map(|c| c.to_ascii_lowercase())+ .filter(|c| !matches!(c, ',' | '.' | '\\' | '=' | '*' | '(' | ')'))+ .collect(),+ )+ .or_insert(0)+ .add_assign(1);+ }++ #[inline]+ pub fn all_probabilities(+ &self,+ term: &str,+ edit_distance: usize,+ ) -> Vec<HashSet<DictionaryResult>> {+ let searcher = fst::automaton::Levenshtein::new(term, edit_distance as u32);+ let mut res = Vec::with_capacity(edit_distance + 1);++ if searcher.is_err() {+ return res;+ }++ for _ in 0..edit_distance + 1 {+ res.push(HashSet::new());+ }++ let searcher = searcher.unwrap();++ let mut matches = self.perform_search(&searcher);+ let dist_metric = LevenshteinDistance::new(edit_distance);++ while let Some((correction, freq)) = matches.next() {+ let correction = std::str::from_utf8(correction).unwrap().to_owned();+ let dist = dist_metric.compare(term, &correction);++ let prob = freq as f64 / self.total_freq as f64;++ res[dist].insert(DictionaryResult { correction, prob });+ }++ res+ }++ fn perform_search<A: Automaton>(&self, searcher: A) -> fst::map::Stream<'_, A> {+ match &self.map {+ InnerMap::File(map) => map.search(searcher).into_stream(),+ InnerMap::Memory(map) => map.search(searcher).into_stream(),+ }+ }++ #[inline]+ pub fn probability(&self, term: &str) -> Option<f64> {+ let searcher = fst::automaton::Str::new(term);+ let mut matches = self.perform_search(&searcher);++ matches+ .next()+ .map(|(_key, frequency)| frequency as f64 / self.total_freq as f64)+ }++ #[cfg(test)]+ pub fn contains(&self, term: &str) -> bool {+ self.probability(term).is_some()+ }++ pub fn insert_page(&mut self, webpage: &Webpage) {+ for text in webpage.html.clean_text().unwrap_or_default().split(' ') {+ self.insert(text);+ }+ }++ pub fn merge(mut self, mut other: Dictionary<TOP_N>) -> Self {+ self.commit().unwrap();+ other.commit().unwrap();++ let map = mem::replace(&mut self.map, InnerMap::Memory(Map::default()));++ let union = match (&map, &other.map) {+ (InnerMap::Memory(map), InnerMap::Memory(other_map)) => map.op().add(other_map).union(),+ (InnerMap::Memory(map), InnerMap::File(other_map)) => map.op().add(other_map).union(),+ (InnerMap::File(map), InnerMap::Memory(other_map)) => map.op().add(other_map).union(),+ (InnerMap::File(map), InnerMap::File(other_map)) => map.op().add(other_map).union(),+ };++ self.store_union(union).unwrap();+ self.total_freq = self.map.total_freq();++ self+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn test_probability() {+ let mut dict = Dictionary::default();++ dict.insert("this");+ dict.insert("is");+ dict.insert("a");+ dict.insert("test");+ dict.insert("test");++ dict.commit().unwrap();++ assert_eq!(dict.probability("this"), Some(0.2));+ assert_eq!(dict.probability("is"), Some(0.2));+ assert_eq!(dict.probability("a"), Some(0.2));+ assert_eq!(dict.probability("test"), Some(0.4));+ }++ #[test]+ fn test_uncontained_term() {+ let mut dict = Dictionary::default();++ dict.insert("this");+ dict.insert("is");+ dict.insert("a");+ dict.insert("test");+ dict.insert("test");++ dict.commit().unwrap();++ assert_eq!(dict.probability("the"), None);++ assert!(dict.contains("this"));+ assert!(dict.contains("is"));+ assert!(dict.contains("test"));+ assert!(!dict.contains("the"));+ }++ #[test]+ fn test_probability_edit_3() {+ let mut dict = Dictionary::default();++ dict.insert("test");+ dict.insert("twst");+ dict.insert("kage");++ dict.commit().unwrap();+ let res = dict.all_probabilities("test", 2);++ let mut num_elements = 0;++ for set in res {+ num_elements += set.len();+ }++ assert_eq!(num_elements, 2);+ }++ #[test]+ fn merge() {+ let mut dict1 = Dictionary::default();++ dict1.insert("test");+ dict1.insert("kage");++ let mut dict2 = Dictionary::default();++ dict2.insert("yay");+ dict2.insert("ay");++ let res = dict1.merge(dict2);++ assert!(res.contains("test"));+ assert!(res.contains("kage"));+ assert!(res.contains("yay"));+ assert!(res.contains("ay"));++ assert_eq!(res.probability("test"), Some(1.0 / 4.0));+ assert_eq!(res.probability("kage"), Some(1.0 / 4.0));+ assert_eq!(res.probability("yay"), Some(1.0 / 4.0));+ assert_eq!(res.probability("ay"), Some(1.0 / 4.0));+ }++ #[test]+ fn only_stop_top_n() {+ let mut dict: Dictionary<2> = Dictionary::open::<&str>(None).unwrap();++ dict.insert("test");+ dict.insert("test");+ dict.insert("hej");+ dict.insert("kage");+ dict.insert("kage");++ dict.commit().unwrap();++ assert!(dict.contains("test"));+ assert!(dict.contains("kage"));+ assert!(!dict.contains("hej"));+ }++ #[test]+ fn weird_characters_ignored() {+ let mut dict = Dictionary::default();++ dict.insert("lennon,");++ dict.commit().unwrap();++ assert!(dict.contains("lennon"));+ assert!(!dict.contains("lennon,"));+ }+}diff --git a/src/spell/distance.rs b/src/spell/distance.rsnew file mode 100644index 00000000..515a0ae4--- /dev/null+++ b/src/spell/distance.rs@@ -0,0 +1,108 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.+use serde::{Deserialize, Serialize};+use std::hash::{Hash, Hasher};++#[derive(Default, Serialize, Deserialize, Debug)]+pub struct LevenshteinDistance {+ dist: usize,+}++impl LevenshteinDistance {+ pub fn new(distance: usize) -> Self {+ LevenshteinDistance { dist: distance }+ }++ /// Function taken from https://github.com/febeling/edit-distance/blob/5597816456e7153cf69092f6ab5d0b4edb5e3797/src/lib.rs#L31+ ///+ /// The [Levenshtein edit distance][wikipedia] between two strings is+ /// the number of individual single-character changes (insert, delete,+ /// substitute) necessary to change string `a` into `b`.+ ///+ /// This can be a used to order search results, for fuzzy+ /// auto-completion, and to find candidates for spelling correction.+ ///+ /// This function returns the edit distance between strings `a` and `b`.+ ///+ /// The runtime complexity is `O(m*n)`, where `m` and `n` are the+ /// strings' lengths.+ pub fn compare(&self, a: &str, b: &str) -> usize {+ let len_a = a.chars().count();+ let len_b = b.chars().count();+ if len_a < len_b {+ return self.compare(b, a);+ }+ // handle special case of 0 length+ if len_a == 0 {+ return len_b;+ } else if len_b == 0 {+ return len_a;+ }++ let len_b = len_b + 1;++ let mut pre;+ let mut tmp;+ let mut cur = vec![0; len_b];++ // initialize string+ for (i, val) in cur.iter_mut().enumerate().take(len_b).skip(1) {+ *val = i;+ }++ // calculate edit distance+ for (i, ca) in a.chars().enumerate() {+ // get first column for this row+ pre = cur[0];+ cur[0] = i + 1;+ for (j, cb) in b.chars().enumerate() {+ tmp = cur[j + 1];+ cur[j + 1] = std::cmp::min(+ // deletion+ tmp + 1,+ std::cmp::min(+ // insertion+ cur[j] + 1,+ // match or substitution+ pre + if ca == cb { 0 } else { 1 },+ ),+ );+ pre = tmp;+ }+ }+ cur[len_b - 1]+ }+}++#[derive(Debug)]+pub struct EditDistance {+ pub dist: usize,+ pub edited_string: String,+}++impl PartialEq for EditDistance {+ fn eq(&self, other: &Self) -> bool {+ self.edited_string == other.edited_string+ }+}++impl Eq for EditDistance {}++impl Hash for EditDistance {+ fn hash<H: Hasher>(&self, state: &mut H) {+ self.edited_string.hash(state);+ }+}diff --git a/src/spell/mod.rs b/src/spell/mod.rsnew file mode 100644index 00000000..3cdf2aec--- /dev/null+++ b/src/spell/mod.rs@@ -0,0 +1,23 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.+pub mod dictionary;+pub mod distance;+pub mod spell_checker;+pub mod splitter;++pub use self::dictionary::{Dictionary, DictionaryResult, EditStrategy, LogarithmicEdit};+pub use self::spell_checker::SpellChecker;+pub use self::splitter::TermSplitter;diff --git a/src/spell/spell_checker.rs b/src/spell/spell_checker.rsnew file mode 100644index 00000000..dfd55d78--- /dev/null+++ b/src/spell/spell_checker.rs@@ -0,0 +1,170 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.+use crate::spell::dictionary::EditStrategy;+use crate::spell::{Dictionary, DictionaryResult};+use std::iter::FromIterator;++pub struct SpellChecker<'a, T: EditStrategy, const DICT_N: usize> {+ dict: &'a Dictionary<DICT_N>,+ edit_strategy: T,+}++impl<'a, T: EditStrategy, const DICT_N: usize> SpellChecker<'a, T, DICT_N> {+ pub fn new(dict: &'a Dictionary<DICT_N>, edit_strategy: T) -> Self {+ SpellChecker {+ dict,+ edit_strategy,+ }+ }++ pub fn correct_top(&self, term: &str, top_n: usize) -> Vec<String> {+ let mut res = Vec::new();++ // this is sorted by increasing edit distance+ for corrections in self+ .dict+ .all_probabilities(term, self.edit_strategy.distance_for_string(term))+ .iter_mut()+ {+ let mut corrections: Vec<&DictionaryResult> = Vec::from_iter(corrections.iter());+ corrections.sort_by(|a, b| b.prob.partial_cmp(&a.prob).unwrap());++ res.extend(+ corrections+ .into_iter()+ .map(|dict_result| dict_result.correction.clone())+ .take(top_n - res.len()),+ );++ if res.len() >= top_n {+ return res;+ }+ }++ res+ }++ pub fn correct(&self, term: &str) -> Option<String> {+ if let Some(correction) = self.correct_top(term, 1).into_iter().next() {+ if correction == term {+ None+ } else {+ Some(correction)+ }+ } else {+ None+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::spell::{dictionary::MaxEdit, LogarithmicEdit};++ #[test]+ fn simple_corrections() {+ let mut dict = Dictionary::default();++ dict.insert("this");+ dict.insert("is");+ dict.insert("a");+ dict.insert("test");+ dict.insert("test");+ dict.insert("pest");++ dict.commit().unwrap();++ let spell_checker = SpellChecker::new(&dict, MaxEdit::new(1));++ assert_eq!(spell_checker.correct("tst"), Some("test".to_string()));+ assert_eq!(spell_checker.correct("ths"), Some("this".to_string()));+ assert_eq!(spell_checker.correct("thes"), Some("this".to_string()));+ assert_eq!(spell_checker.correct("is"), None);+ assert_eq!(+ spell_checker.correct_top("thes", 3),+ vec!["this".to_string()]+ );+ assert_eq!(+ spell_checker.correct_top("dest", 3),+ vec!["test".to_string(), "pest".to_string()]+ );+ assert_eq!(+ spell_checker.correct_top("dest", 1),+ vec!["test".to_string()]+ );+ }++ #[test]+ fn correct_uncontained_word() {+ let mut dict = Dictionary::default();++ dict.insert("this");+ dict.insert("is");+ dict.insert("a");+ dict.insert("test");+ dict.insert("test");++ dict.commit().unwrap();++ let spell_checker = SpellChecker::new(&dict, MaxEdit::new(1));++ assert_eq!(spell_checker.correct("what"), None);+ assert!(spell_checker.correct_top("what", 2).is_empty());+ }++ #[test]+ fn prioritise_low_distance_words() {+ let mut dict = Dictionary::default();++ dict.insert("this");+ dict.insert("is");+ dict.insert("a");+ dict.insert("test");+ dict.insert("contest");+ dict.insert("contest");++ dict.commit().unwrap();++ let spell_checker = SpellChecker::new(&dict, MaxEdit::new(4));++ assert_eq!(spell_checker.correct("dest"), Some("test".to_string()));+ assert_eq!(+ spell_checker.correct_top("dest", 3),+ vec!["test".to_string(), "is".to_string(), "contest".to_string()]+ );+ }++ #[test]+ fn correct_sorting_multiple_hits() {+ let mut dict = Dictionary::default();++ dict.insert("the");+ dict.insert("the");+ dict.insert("the");+ dict.insert("he");++ dict.commit().unwrap();++ let spell_checker = SpellChecker::new(&dict, LogarithmicEdit::new(4));++ assert_eq!(spell_checker.correct("fhe"), Some("the".to_string()));+ assert_eq!(+ spell_checker.correct_top("fhe", 3),+ vec!["the".to_string(), "he".to_string()]+ );+ }+}diff --git a/src/spell/splitter.rs b/src/spell/splitter.rsnew file mode 100644index 00000000..876d098f--- /dev/null+++ b/src/spell/splitter.rs@@ -0,0 +1,108 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.+use crate::spell::Dictionary;++pub struct TermSplitter<'a, const DICT_N: usize> {+ dict: &'a Dictionary<DICT_N>,+}++impl<'a, const DICT_N: usize> TermSplitter<'a, DICT_N> {+ pub fn new(dict: &'a Dictionary<DICT_N>) -> Self {+ TermSplitter { dict }+ }+ pub fn split(&self, text: &'a str) -> Vec<&'a str> {+ let mut probs = vec![1.0];+ let mut lasts: Vec<usize> = vec![0];++ for i in 1..text.len() + 1 {+ let mut best_prob_k = 0.0;+ let mut best_index = 0;++ for j in 0..i {+ if !text.is_char_boundary(j) || !text.is_char_boundary(i) {+ continue;+ }++ if let Some(prob) = &self.dict.probability(&text[j..i]) {+ let new_prob = probs[j] * prob;+ if new_prob > best_prob_k {+ best_prob_k = new_prob;+ best_index = j;+ }+ }+ }++ probs.push(best_prob_k);+ lasts.push(best_index);+ }++ let mut words = Vec::new();+ let mut i = text.len();++ while i > 0 {+ if i == text.len() && lasts[i] == 0 {+ break;+ }+ words.push(&text[lasts[i]..i]);+ i = lasts[i];+ }++ words.reverse();+ words+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn test_split() {+ let mut dictionary = Dictionary::default();++ dictionary.insert("wicked");+ dictionary.insert("weather");++ dictionary.commit().unwrap();++ assert_eq!(+ TermSplitter::new(&dictionary).split("wickedweather"),+ vec!["wicked", "weather"]+ );++ assert_eq!(TermSplitter::new(&dictionary).split("wicked").len(), 0);++ assert_eq!(TermSplitter::new(&dictionary).split("udlæg").len(), 0);+ }++ #[test]+ fn test_most_probable() {+ let mut dictionary = Dictionary::default();++ dictionary.insert("wicked");+ dictionary.insert("wicked");+ dictionary.insert("weather");+ dictionary.insert("weather");+ dictionary.insert("eat"); // "eat" is a substring of "weather"++ dictionary.commit().unwrap();++ assert_eq!(+ TermSplitter::new(&dictionary).split("wickedweather"),+ vec!["wicked", "weather"]+ );+ }+}diff --git a/templates/search.html b/templates/search.htmlindex 112fcc94..9144010e 100644--- a/templates/search.html+++ b/templates/search.html@@ -15,45 +15,57 @@</div><div class="flex pt-8">- <div class="search-result flex flex-col space-y-3 pb-8 pl-40">- {% for item in search_result %}- <div class="flex max-w-3xl w-full p-3">- <div class="flex flex-col w-full">- <div class="text-sm mb-1 flex">- <div class="flex items-center mr-2">- <img src="/favicons/{{ item.domain }}" width="13px" />+ <div class="flex flex-col pb-8 pl-40">+ {% match spell_correction %}+ {% when Some with (correction) %}+ <div class="pl-3 pb-3 ">+ Did you mean: <a class="italic title-link font-bold" href="/search?q={{ correction }}">{{ correction+ }}</a>+ </div>+ {% when None %}+ {% endmatch %}++ <div class="search-result flex flex-col space-y-3">+ {% for item in search_result %}+ <div class="flex max-w-3xl w-full p-3">+ <div class="flex flex-col w-full">+ <div class="text-sm mb-1 flex">+ <div class="flex items-center mr-2">+ <img src="/favicons/{{ item.domain }}" width="13px" />+ </div>+ <div>+ <a class="hover:no-underline text-gray-600" href="{{ item.url }}">{{ item.pretty_url+ }}</a>+ </div></div>- <div>- <a class="hover:no-underline text-gray-600" href="{{ item.url }}">{{ item.pretty_url }}</a>+ <div class="text-lg font-medium mb-1">+ <a class="title-link" href="{{ item.url }}">{{ item.title }}</a>+ </div>+ <div class="text-sm">+ {% match item.last_updated %}+ {% when Some with (val) %}+ <span class="text-gray-500">+ {{ val }}+ </span> —+ {% when None %}+ {% endmatch %}+ {{ item.snippet }}</div></div>- <div class="text-lg font-medium mb-1">- <a class="title-link" href="{{ item.url }}">{{ item.title }}</a>- </div>- <div class="text-sm">- {% match item.last_updated %}- {% when Some with (val) %}- <span class="text-gray-500">- {{ val }}- </span> —- {% when None %}- {% endmatch %}- {{ item.snippet }}- </div>- </div>- <div class="flex h-full items-center">- <div class="h-20 w-20 pl-2 pt-1 pb-1">- {% match item.primary_image_uuid %}- {% when Some with (val) %}- <a href="{{ item.url }}">- <img class="h-full w-full object-cover rounded-full" src="/image/{{ val }}" />- </a>- {% when None %}- {% endmatch %}+ <div class="flex h-full items-center">+ <div class="h-20 w-20 pl-2 pt-1 pb-1">+ {% match item.primary_image_uuid %}+ {% when Some with (val) %}+ <a href="{{ item.url }}">+ <img class="h-full w-full object-cover rounded-full" src="/image/{{ val }}" />+ </a>+ {% when None %}+ {% endmatch %}+ </div></div></div>+ {% endfor %}</div>- {% endfor %}</div>{% match entity %}