Skip to content

web search done right

Ftr/spellcheck (#14)

Mikkel Denkerc5ec4f46f15c
diff --git a/Cargo.lock b/Cargo.lock
index b11a824e..2ac2347d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -582,6 +582,7 @@ dependencies = [
"lru",
"maplit",
"md5",
+ "memmap",
"nom",
"parse_wiki_text",
"quick-xml",
@@ -797,6 +798,9 @@ name = "fst"
version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ab85b9b05e3978cc9a9cf8fea7f01b494e1a09ed3037e16ba39edc7a29eb61a"
+dependencies = [
+ "utf8-ranges",
+]
[[package]]
name = "futures"
@@ -1444,6 +1448,16 @@ version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
+[[package]]
+name = "memmap"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b"
+dependencies = [
+ "libc",
+ "winapi",
+]
+
[[package]]
name = "memmap2"
version = "0.5.4"
diff --git a/Cargo.toml b/Cargo.toml
index 666e6e88..ae349940 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -41,7 +41,7 @@ axum-extra = { version = "0.3.4", features = ["spa"] }
html-escape = "0.2.11"
logos = "0.12.1"
csv = "1.1.6"
-fst = "0.4.7"
+fst = { version = "0.4.7", features = ["levenshtein"] }
lazy_static = "1.4.0"
rocksdb = "0.18.0"
image = "0.24.3"
@@ -52,6 +52,7 @@ quick-xml = "0.23.0"
bzip2 = "0.4.3"
parse_wiki_text = "0.1.5"
md5 = "0.7.0"
+memmap = "0.7.0"
[dev-dependencies]
maplit = "1.0.2"
diff --git a/data/entity/attribute_occurrences/OPTIONS-000135 b/data/entity/attribute_occurrences/OPTIONS-000135
deleted file mode 100644
index 9f62adb9..00000000
--- a/data/entity/attribute_occurrences/OPTIONS-000135
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9d110b3864bbce0573ace88015c3b8da37878518c24f398ba98390ffe012efdc
-size 6535
diff --git a/data/entity/images/OPTIONS-000147 b/data/entity/images/OPTIONS-000147
deleted file mode 100644
index 9f62adb9..00000000
--- a/data/entity/images/OPTIONS-000147
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9d110b3864bbce0573ace88015c3b8da37878518c24f398ba98390ffe012efdc
-size 6535
diff --git a/data/index/favicon_store/000009.sst b/data/index/favicon_store/000009.sst
index e35b63eb..62f314de 100644
--- a/data/index/favicon_store/000009.sst
+++ b/data/index/favicon_store/000009.sst
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:8ddc6d4940e8aef7aceb41a85e47e10919bc2ceeb424b2e0e461549ca5d44d61
-size 89167
+oid sha256:fa3cad28e7129c270895f1f2376ce5e635036e7d3b632106d5109bc59f106be2
+size 133796
diff --git a/data/index/favicon_store/IDENTITY b/data/index/favicon_store/IDENTITY
index c5717c62..0eea3b44 100644
--- a/data/index/favicon_store/IDENTITY
+++ b/data/index/favicon_store/IDENTITY
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:740eb887bf29fb004308c69e067b676c0f54ace10fb5f678c0ea08a2496d95ba
+oid sha256:d13bef3c0e708ed912ebf554c23fdb705667714a71092dffd659d562875b9729
size 36
diff --git a/data/index/favicon_store/LOG.old.1660643834782411 b/data/index/favicon_store/LOG.old.1660643834782411
deleted file mode 100644
index 10b796ba..00000000
--- a/data/index/favicon_store/LOG.old.1660643834782411
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b5c482c1291fb84b3d412a0036dcdfbbed93aa5255236135b4a074c4ba5bbcf5
-size 26146
diff --git a/data/index/favicon_store/LOG.old.1660643841806793 b/data/index/favicon_store/LOG.old.1660643841806793
deleted file mode 100644
index fcbcaa0a..00000000
--- a/data/index/favicon_store/LOG.old.1660643841806793
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:fc85b5da8d22a07cff04ee71d6f2248c6b1151ea61e6aec34e7b238bd3f25473
-size 20225
diff --git a/data/index/favicon_store/LOG.old.1660643992979668 b/data/index/favicon_store/LOG.old.1660643992979668
deleted file mode 100644
index 4d8833a2..00000000
--- a/data/index/favicon_store/LOG.old.1660643992979668
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:357c4622739edd1087d52615a0dec7196f6e48b0d7d92af9c0924a1f324dfb68
-size 22900
diff --git a/data/index/favicon_store/LOG.old.1660644125900052 b/data/index/favicon_store/LOG.old.1660644125900052
deleted file mode 100644
index c356d66c..00000000
--- a/data/index/favicon_store/LOG.old.1660644125900052
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:64f1021d417aa828cdcc6c45651bb41e557f13c444be1aa1bd8b66a8a9d2a6d3
-size 22902
diff --git a/data/index/favicon_store/LOG.old.1660644236142610 b/data/index/favicon_store/LOG.old.1660644236142610
deleted file mode 100644
index b9a7b2ac..00000000
--- a/data/index/favicon_store/LOG.old.1660644236142610
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3c7b70ac6206396332085efc106fcc046cec6e8e8fbe0223604969c62c863b3c
-size 22935
diff --git a/data/index/favicon_store/LOG.old.1660644357270366 b/data/index/favicon_store/LOG.old.1660644357270366
deleted file mode 100644
index 43c036b1..00000000
--- a/data/index/favicon_store/LOG.old.1660644357270366
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3a4e1ae4f35e359c1f5ea4ffc11ab64aed7df14a69d2b4bcad3106023ac1cab7
-size 22934
diff --git a/data/index/favicon_store/LOG.old.1660649840289268 b/data/index/favicon_store/LOG.old.1660649840289268
deleted file mode 100644
index 477666cb..00000000
--- a/data/index/favicon_store/LOG.old.1660649840289268
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:42ab28c8deaa48ffbf17f896868db0c80a34981c20560869065b4fef9e970aad
-size 22934
diff --git a/data/index/favicon_store/LOG.old.1660649960425303 b/data/index/favicon_store/LOG.old.1660649960425303
deleted file mode 100644
index 46ca663b..00000000
--- a/data/index/favicon_store/LOG.old.1660649960425303
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:2694cdc5c84a20196b53e991ae2ee5c27553a61706def5c57efba32cd1268b99
-size 22935
diff --git a/data/index/favicon_store/LOG.old.1660650026549335 b/data/index/favicon_store/LOG.old.1660650026549335
deleted file mode 100644
index a6342ccc..00000000
--- a/data/index/favicon_store/LOG.old.1660650026549335
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ba6daeaa11629123e9bd7b16b9f6e1218d6e1eb2178dde94ed2d1a9c6c974e73
-size 22934
diff --git a/data/index/favicon_store/LOG.old.1660650103901274 b/data/index/favicon_store/LOG.old.1660650103901274
deleted file mode 100644
index f6c3d3ed..00000000
--- a/data/index/favicon_store/LOG.old.1660650103901274
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:46bc5f5de71273bc90a677d403e1cc0abf5056f7c74d2794a56022e6a25c269c
-size 22932
diff --git a/data/index/favicon_store/LOG.old.1660651249991160 b/data/index/favicon_store/LOG.old.1660651249991160
deleted file mode 100644
index 84b321b4..00000000
--- a/data/index/favicon_store/LOG.old.1660651249991160
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:bc049602a3cafe817c6e1b0bf75a9021ffbf3a1b6ae372f541270cf85bda40f5
-size 22933
diff --git a/data/index/favicon_store/OPTIONS-000161 b/data/index/favicon_store/OPTIONS-000161
deleted file mode 100644
index 9f62adb9..00000000
--- a/data/index/favicon_store/OPTIONS-000161
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9d110b3864bbce0573ace88015c3b8da37878518c24f398ba98390ffe012efdc
-size 6535
diff --git a/data/index/inverted_index/.managed.json b/data/index/inverted_index/.managed.json
index e9787a55..1ba9113d 100644
--- a/data/index/inverted_index/.managed.json
+++ b/data/index/inverted_index/.managed.json
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:7e3a2db879d6be2cd18aa86e23e3dd9717c565ca4383b396de978d50c11f27c9
+oid sha256:6fc73ac5556fab2417af3b089d9e51b0475ba3308f99153982fd3eb2c63f9d1e
size 258
diff --git a/data/index/inverted_index/381ccf14f7454ae6adf96a643f363ce8.fast b/data/index/inverted_index/381ccf14f7454ae6adf96a643f363ce8.fast
deleted file mode 100644
index ab585503..00000000
--- a/data/index/inverted_index/381ccf14f7454ae6adf96a643f363ce8.fast
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f7bff4b7235b7d1e87d365551963e4a24756bbaef5abbfb1ce4d274a19377610
-size 222787
diff --git a/data/index/inverted_index/381ccf14f7454ae6adf96a643f363ce8.fieldnorm b/data/index/inverted_index/381ccf14f7454ae6adf96a643f363ce8.fieldnorm
deleted file mode 100644
index d94c7d3e..00000000
--- a/data/index/inverted_index/381ccf14f7454ae6adf96a643f363ce8.fieldnorm
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:8ce278866b99fcd5bbe712a48185107b1d7d330d6b1185ecec2f3cee83ddd483
-size 318674
diff --git a/data/index/inverted_index/381ccf14f7454ae6adf96a643f363ce8.idx b/data/index/inverted_index/381ccf14f7454ae6adf96a643f363ce8.idx
deleted file mode 100644
index cf9e9074..00000000
--- a/data/index/inverted_index/381ccf14f7454ae6adf96a643f363ce8.idx
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:82045379ae39b536f09e46b22987f4b6991d26c2066649994253dab50792115b
-size 54219587
diff --git a/data/index/inverted_index/381ccf14f7454ae6adf96a643f363ce8.pos b/data/index/inverted_index/381ccf14f7454ae6adf96a643f363ce8.pos
deleted file mode 100644
index 24f1180b..00000000
--- a/data/index/inverted_index/381ccf14f7454ae6adf96a643f363ce8.pos
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:aa5f3acd93c379b463ce4e732d5c9302f377cc890bd201227a8ea71d4dca1e7d
-size 111367468
diff --git a/data/index/inverted_index/381ccf14f7454ae6adf96a643f363ce8.store b/data/index/inverted_index/381ccf14f7454ae6adf96a643f363ce8.store
deleted file mode 100644
index c2777e07..00000000
--- a/data/index/inverted_index/381ccf14f7454ae6adf96a643f363ce8.store
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:511e6fb2d03a58bc14408bc2188192953ac69561b6a84dca18a1279ed32741b7
-size 108014845
diff --git a/data/index/inverted_index/381ccf14f7454ae6adf96a643f363ce8.term b/data/index/inverted_index/381ccf14f7454ae6adf96a643f363ce8.term
deleted file mode 100644
index f8f7402f..00000000
--- a/data/index/inverted_index/381ccf14f7454ae6adf96a643f363ce8.term
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a99487b51229045bde17902e2ea3c12e1c2e4d3b0ca88257425c6b44c099facd
-size 85274891
diff --git a/data/index/inverted_index/8842c700af704f7db01221862b8fee35.fast b/data/index/inverted_index/8842c700af704f7db01221862b8fee35.fast
new file mode 100644
index 00000000..74692838
--- /dev/null
+++ b/data/index/inverted_index/8842c700af704f7db01221862b8fee35.fast
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55c9745319da0509b16c0cc0d2170b76168ff88b24e1cf64b80073d578734a05
+size 224771
diff --git a/data/index/inverted_index/8842c700af704f7db01221862b8fee35.fieldnorm b/data/index/inverted_index/8842c700af704f7db01221862b8fee35.fieldnorm
new file mode 100644
index 00000000..e97b8610
--- /dev/null
+++ b/data/index/inverted_index/8842c700af704f7db01221862b8fee35.fieldnorm
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dfd123ad020d5f2d56a5ee465115aba35ada8d44e26407a37e8da69a2ad171df
+size 318675
diff --git a/data/index/inverted_index/8842c700af704f7db01221862b8fee35.idx b/data/index/inverted_index/8842c700af704f7db01221862b8fee35.idx
new file mode 100644
index 00000000..62b2e139
--- /dev/null
+++ b/data/index/inverted_index/8842c700af704f7db01221862b8fee35.idx
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6151ebd74b0b9f557d6ec30b521aa3c980931c722b2ac93b55280045a162828f
+size 54160567
diff --git a/data/index/inverted_index/8842c700af704f7db01221862b8fee35.pos b/data/index/inverted_index/8842c700af704f7db01221862b8fee35.pos
new file mode 100644
index 00000000..568f2c51
--- /dev/null
+++ b/data/index/inverted_index/8842c700af704f7db01221862b8fee35.pos
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1be7c947556c915dc2eef0a3b78e4767e8b70c8a28b7c940eba157e895c8b1ed
+size 111365445
diff --git a/data/index/inverted_index/8842c700af704f7db01221862b8fee35.store b/data/index/inverted_index/8842c700af704f7db01221862b8fee35.store
new file mode 100644
index 00000000..0765e6fe
--- /dev/null
+++ b/data/index/inverted_index/8842c700af704f7db01221862b8fee35.store
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c9ae983092355587f2993ebefe9bc8c03b8d72e105bdf5d15b579b89cca48f9
+size 108022084
diff --git a/data/index/inverted_index/8842c700af704f7db01221862b8fee35.term b/data/index/inverted_index/8842c700af704f7db01221862b8fee35.term
new file mode 100644
index 00000000..0066651f
--- /dev/null
+++ b/data/index/inverted_index/8842c700af704f7db01221862b8fee35.term
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28842f363d110638b3673d229576ab0488fcb9c0819af87bf12dc3136a722a46
+size 85271351
diff --git a/data/index/inverted_index/meta.json b/data/index/inverted_index/meta.json
index f2841b5a..cce3f738 100644
--- a/data/index/inverted_index/meta.json
+++ b/data/index/inverted_index/meta.json
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:715b50d87576bfb1794a476727b33f2377810df5b7e5a4719951c814940548a1
+oid sha256:3c2c0ad7e61da59eddc5a42053bca7f84e85ae8f600cde7b10046c2f3c7d7e54
size 4541
diff --git a/data/index/primary_image_store/000009.sst b/data/index/primary_image_store/000009.sst
index dacaa1f5..eefd8bd9 100644
--- a/data/index/primary_image_store/000009.sst
+++ b/data/index/primary_image_store/000009.sst
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:0a277ac634dbb9632f40f8e05abf5264bb621238d8a93ac6117888449f4bba36
-size 26218869
+oid sha256:a846b78278d422997fd1e9cc7849f10d2bdf4d3dc320436f8064d0c78921d778
+size 26538187
diff --git a/data/index/primary_image_store/IDENTITY b/data/index/primary_image_store/IDENTITY
index 51bbb310..4095860a 100644
--- a/data/index/primary_image_store/IDENTITY
+++ b/data/index/primary_image_store/IDENTITY
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:ea55c46eebd189a9be90f18ef25a3caea033fd2c5b82ee2eb26adc57bd2bdd80
+oid sha256:766cac6c9a3d17b8a1d67c1b5da99318619dfa21c3aca032c01fcdb3d4547463
size 36
diff --git a/data/index/primary_image_store/LOG.old.1660643834784950 b/data/index/primary_image_store/LOG.old.1660643834784950
deleted file mode 100644
index 78329b7b..00000000
--- a/data/index/primary_image_store/LOG.old.1660643834784950
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c055c05cbfd1b9915334d940e0c0a4113c800d1583a61de40fef492db0e8a322
-size 26213
diff --git a/data/index/primary_image_store/LOG.old.1660643841811016 b/data/index/primary_image_store/LOG.old.1660643841811016
deleted file mode 100644
index e34353a7..00000000
--- a/data/index/primary_image_store/LOG.old.1660643841811016
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d7e99db95ec0481e6d9e6d19ce6f92577e70c81a301fd7b80914f56c7db9611d
-size 20257
diff --git a/data/index/primary_image_store/LOG.old.1660643992984221 b/data/index/primary_image_store/LOG.old.1660643992984221
deleted file mode 100644
index 7b5b4ebd..00000000
--- a/data/index/primary_image_store/LOG.old.1660643992984221
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:115073258401b92e1aacfb6ae188e078355bedb0a41c8bccd1d8234b5c56f8aa
-size 22931
diff --git a/data/index/primary_image_store/LOG.old.1660644125904383 b/data/index/primary_image_store/LOG.old.1660644125904383
deleted file mode 100644
index f224e3cd..00000000
--- a/data/index/primary_image_store/LOG.old.1660644125904383
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:8c4dc0ced0085e73914b06749b60e05cdb67c8d116f726cf35d88172a07e56f2
-size 22934
diff --git a/data/index/primary_image_store/LOG.old.1660644236146938 b/data/index/primary_image_store/LOG.old.1660644236146938
deleted file mode 100644
index aa1b14b3..00000000
--- a/data/index/primary_image_store/LOG.old.1660644236146938
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f37a2cc4a9e0b9218fb5f4c4be27e5dd630fd00eddbaf3c263bba94cb36fe553
-size 22967
diff --git a/data/index/primary_image_store/LOG.old.1660644357272670 b/data/index/primary_image_store/LOG.old.1660644357272670
deleted file mode 100644
index d17f7cf9..00000000
--- a/data/index/primary_image_store/LOG.old.1660644357272670
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:1bff841f7bf97e4b3f3b6a9234aa826257012cb758f9a49e59fa5c4cd436397f
-size 22966
diff --git a/data/index/primary_image_store/LOG.old.1660649840294017 b/data/index/primary_image_store/LOG.old.1660649840294017
deleted file mode 100644
index 6d4ce341..00000000
--- a/data/index/primary_image_store/LOG.old.1660649840294017
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:51f43612649838caba68b88842d858da69c6f700febd0194b1ad260cc2791650
-size 22963
diff --git a/data/index/primary_image_store/LOG.old.1660649960427655 b/data/index/primary_image_store/LOG.old.1660649960427655
deleted file mode 100644
index 0fc76d3f..00000000
--- a/data/index/primary_image_store/LOG.old.1660649960427655
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:86fb7ab2e29c2f8e5103abfbc9711f8e7fcfb672a7a2669538d320258ee33f94
-size 22967
diff --git a/data/index/primary_image_store/LOG.old.1660650026552591 b/data/index/primary_image_store/LOG.old.1660650026552591
deleted file mode 100644
index 22163210..00000000
--- a/data/index/primary_image_store/LOG.old.1660650026552591
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:cf9c379fd750cbe54aa0e46beaa351c7ea4ad0f21856f1449de52868945f73f5
-size 22965
diff --git a/data/index/primary_image_store/LOG.old.1660650103903637 b/data/index/primary_image_store/LOG.old.1660650103903637
deleted file mode 100644
index 5f5a20a2..00000000
--- a/data/index/primary_image_store/LOG.old.1660650103903637
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:0056c77eeb1707d7f68ed856929332c7870724dc50bb7a773abdbb32b4bdb127
-size 22965
diff --git a/data/index/primary_image_store/LOG.old.1660651249995469 b/data/index/primary_image_store/LOG.old.1660651249995469
deleted file mode 100644
index d847ede1..00000000
--- a/data/index/primary_image_store/LOG.old.1660651249995469
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:00d6d4157444dc238cfdef8ee130778ba2c67d5b5c6160ec25b8641a3136b95c
-size 22933
diff --git a/data/index/primary_image_store/OPTIONS-000161 b/data/index/primary_image_store/OPTIONS-000161
deleted file mode 100644
index 9f62adb9..00000000
--- a/data/index/primary_image_store/OPTIONS-000161
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9d110b3864bbce0573ace88015c3b8da37878518c24f398ba98390ffe012efdc
-size 6535
diff --git a/data/index/primary_image_store/dictionary b/data/index/primary_image_store/dictionary
new file mode 100644
index 00000000..2452fa9d
--- /dev/null
+++ b/data/index/primary_image_store/dictionary
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ffc57eb861fab46f5c16cffbe9a692a1beb072a1282bd325562a3a90928b9367
+size 5141310
diff --git a/src/frontend/search.rs b/src/frontend/search.rs
index 022d8897..c39d9238 100644
--- a/src/frontend/search.rs
+++ b/src/frontend/search.rs
@@ -234,6 +234,7 @@ struct SearchTemplate {
search_result: Vec<DisplayedWebpage>,
query: String,
entity: Option<DisplayedEntity>,
+ spell_correction: Option<String>,
}
pub async fn route(
@@ -243,6 +244,7 @@ pub async fn route(
let mut search_result = Vec::new();
let mut displayed_query = String::new();
let mut entity = None;
+ let mut spell_correction = None;
if let Some(query) = params.get("q") {
displayed_query = query.clone();
@@ -269,12 +271,14 @@ pub async fn route(
entity = result
.entity
.map(|entity| DisplayedEntity::from(entity, &state.searcher));
+ spell_correction = result.spell_corrected_query;
}
let template = SearchTemplate {
search_result,
query: displayed_query,
entity,
+ spell_correction,
};
HtmlTemplate(template)
}
diff --git a/src/index.rs b/src/index.rs
index ea9b7f42..ecc9bcae 100644
--- a/src/index.rs
+++ b/src/index.rs
@@ -20,6 +20,7 @@ use std::path::Path;
use std::sync::Arc;
use std::time::Duration;
+use itertools::intersperse;
use serde::{Deserialize, Serialize};
use tantivy::collector::Collector;
use tantivy::schema::Schema;
@@ -30,12 +31,14 @@ use crate::image_downloader::{ImageDownloadJob, ImageDownloader};
use crate::image_store::{FaviconStore, Image, ImageStore, PrimaryImageStore};
use crate::inverted_index::{InvertedIndex, InvertedIndexSearchResult};
use crate::query::Query;
+use crate::spell::{Dictionary, LogarithmicEdit, SpellChecker, TermSplitter};
use crate::webpage::{Url, Webpage};
use crate::Result;
const INVERTED_INDEX_SUBFOLDER_NAME: &str = "inverted_index";
const FAVICON_STORE_SUBFOLDER_NAME: &str = "favicon_store";
const PRIMARY_IMAGE_STORE_SUBFOLDER_NAME: &str = "primary_image_store";
+const SPELL_SUBFOLDER_NAME: &str = "primary_image_store";
const IMAGE_WEBPAGE_CENTRALITY_THRESHOLD: f64 = 0.0;
pub struct Index {
@@ -44,6 +47,7 @@ pub struct Index {
primary_image_store: PrimaryImageStore,
favicon_downloader: ImageDownloader<String>,
primary_image_downloader: ImageDownloader<Uuid>,
+ spell_dictionary: Dictionary<1_000_000>,
pub path: String,
}
@@ -66,6 +70,7 @@ impl Index {
primary_image_store,
primary_image_downloader: ImageDownloader::new(),
favicon_downloader: ImageDownloader::new(),
+ spell_dictionary: Dictionary::open(Some(path.as_ref().join(SPELL_SUBFOLDER_NAME)))?,
path: path.as_ref().to_str().unwrap().to_string(),
})
}
@@ -79,11 +84,13 @@ impl Index {
pub fn insert(&mut self, mut webpage: Webpage) -> Result<()> {
self.maybe_insert_favicon(&webpage);
self.maybe_insert_primary_image(&mut webpage);
+ self.spell_dictionary.insert_page(&webpage);
self.inverted_index.insert(webpage)
}
pub fn commit(&mut self) -> Result<()> {
self.inverted_index.merge_all_segments()?;
+ self.spell_dictionary.commit()?;
self.inverted_index.commit()
}
@@ -103,6 +110,8 @@ impl Index {
self.primary_image_store.merge(other.primary_image_store);
drop(self.primary_image_store);
+ self.spell_dictionary.merge(other.spell_dictionary);
+
Self::open(&self.path).expect("failed to open index")
}
@@ -162,6 +171,61 @@ impl Index {
self.primary_image_downloader
.download(&mut self.primary_image_store);
}
+
+ fn spell_check(&self, terms: &[String]) -> Option<String> {
+ let spellchecker = SpellChecker::new(&self.spell_dictionary, LogarithmicEdit::new(4));
+ let mut corrections: Vec<String> = Vec::new();
+
+ for term in terms {
+ match spellchecker.correct(term.to_ascii_lowercase().as_str()) {
+ Some(correction) => corrections.push(correction.to_ascii_lowercase()),
+ None => corrections.push(term.to_ascii_lowercase()),
+ }
+ }
+
+ if corrections
+ .iter()
+ .cloned()
+ .zip(terms.iter().map(|term| term.to_ascii_lowercase()))
+ .all(|(correction, term)| correction == term)
+ {
+ None
+ } else {
+ Some(intersperse(corrections.into_iter(), " ".to_string()).collect())
+ }
+ }
+
+ fn split_words(&self, terms: &[String]) -> Option<String> {
+ let splitter = TermSplitter::new(&self.spell_dictionary);
+ let mut corrections: Vec<String> = Vec::new();
+
+ for term in terms {
+ let t = term.to_ascii_lowercase();
+ let split = splitter.split(t.as_str());
+ if split.is_empty() {
+ corrections.push(term.clone());
+ } else {
+ for s in split {
+ corrections.push(s.to_string())
+ }
+ }
+ }
+
+ if corrections
+ .iter()
+ .cloned()
+ .zip(terms.iter().map(|term| term.to_ascii_lowercase()))
+ .all(|(correction, term)| correction == term)
+ {
+ None
+ } else {
+ Some(intersperse(corrections.into_iter(), " ".to_string()).collect())
+ }
+ }
+
+ pub fn spell_correction(&self, terms: &[String]) -> Option<String> {
+ self.spell_check(terms).or_else(|| self.split_words(terms))
+ }
}
#[derive(Debug, Serialize, Deserialize)]
@@ -234,6 +298,8 @@ mod tests {
))
.expect("failed to parse webpage");
+ index.commit().unwrap();
+
let path = index.path.clone();
let frozen: FrozenIndex = index.into();
let bytes = bincode::serialize(&frozen).unwrap();
@@ -251,5 +317,47 @@ mod tests {
assert_eq!(result.num_docs, 1);
assert_eq!(result.documents.len(), 1);
assert_eq!(result.documents[0].url, "https://www.example.com");
+
+ assert_eq!(
+ index.spell_correction(&["thiss".to_string()]),
+ Some("this".to_string())
+ );
+ }
+
+ #[test]
+ fn sentence_spell_correction() {
+ let mut index = Index::temporary().expect("Unable to open index");
+
+ index
+ .insert(Webpage::new(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ {CONTENT}
+ </body>
+ </html>
+ "#
+ ),
+ "https://www.example.com",
+ vec![],
+ 1.0,
+ 500,
+ ))
+ .expect("failed to parse webpage");
+
+ index.commit().unwrap();
+
+ assert_eq!(
+ index.spell_correction(&["th".to_string(), "best".to_string()]),
+ Some("the best".to_string())
+ );
+ assert_eq!(
+ index.spell_correction(&["the".to_string(), "best".to_string()]),
+ None
+ );
}
}
diff --git a/src/lib.rs b/src/lib.rs
index 107ea854..97838062 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -13,6 +13,9 @@
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+// #![warn(clippy::pedantic)]
+
use serde::{Deserialize, Serialize};
use std::fs::File;
use std::io::{self, BufRead};
@@ -41,6 +44,7 @@ mod schema;
mod schema_org;
pub mod searcher;
mod snippet;
+mod spell;
mod tokenizer;
mod warc;
pub mod webgraph;
@@ -178,6 +182,9 @@ pub enum Error {
#[error("XML parser error")]
XML(#[from] quick_xml::Error),
+
+ #[error("Spell dictionary error")]
+ Spell(#[from] crate::spell::dictionary::DictionaryError),
}
pub(crate) type Result<T> = std::result::Result<T, Error>;
diff --git a/src/query/mod.rs b/src/query/mod.rs
index 73d4d9b2..34ea52df 100644
--- a/src/query/mod.rs
+++ b/src/query/mod.rs
@@ -92,6 +92,10 @@ impl Query {
Err(error) => Err(Error::ParsingError(error.to_string())),
}
}
+
+ pub fn simple_terms(&self) -> &[String] {
+ &self.terms
+ }
}
impl tantivy::query::Query for Query {
diff --git a/src/searcher.rs b/src/searcher.rs
index 8e43e832..8b1d11d3 100644
--- a/src/searcher.rs
+++ b/src/searcher.rs
@@ -29,6 +29,7 @@ use crate::Result;
#[derive(Debug)]
pub struct SearchResult {
+ pub spell_corrected_query: Option<String>,
pub webpages: InvertedIndexSearchResult,
pub entity: Option<StoredEntity>,
}
@@ -59,13 +60,18 @@ impl Searcher {
let query = Query::parse(query, self.index.schema())?;
let ranker = Ranker::new(query.clone());
let webpages = self.index.search(&query, ranker.collector())?;
+ let correction = self.index.spell_correction(query.simple_terms());
let entity = self
.entity_index
.as_ref()
.and_then(|index| index.search(raw_query));
- Ok(SearchResult { webpages, entity })
+ Ok(SearchResult {
+ webpages,
+ entity,
+ spell_corrected_query: correction,
+ })
}
pub fn favicon(&self, site: &Url) -> Option<Image> {
diff --git a/src/spell/dictionary.rs b/src/spell/dictionary.rs
new file mode 100644
index 00000000..74660a96
--- /dev/null
+++ b/src/spell/dictionary.rs
@@ -0,0 +1,475 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use crate::spell::distance::LevenshteinDistance;
+use crate::webpage::Webpage;
+use fst::map::Union;
+use fst::{Automaton, IntoStreamer, Map, MapBuilder, Streamer};
+use memmap::Mmap;
+use std::collections::{BTreeMap, BinaryHeap, HashSet};
+use std::fs::{self, File, OpenOptions};
+use std::hash::{Hash, Hasher};
+use std::ops::AddAssign;
+use std::path::Path;
+use std::{cmp, io, mem};
+use thiserror::Error;
+
+pub trait EditStrategy: Send + Sync {
+ fn distance_for_string(&self, string: &str) -> usize;
+ fn dist(&self) -> LevenshteinDistance;
+}
+
+pub struct LogarithmicEdit {
+ max_edit_distance: usize,
+}
+
+impl LogarithmicEdit {
+ pub fn new(max_edit_distance: usize) -> Self {
+ Self { max_edit_distance }
+ }
+}
+
+impl EditStrategy for LogarithmicEdit {
+ fn distance_for_string(&self, string: &str) -> usize {
+ let log_value: usize = (string.len() as f32).log2() as usize;
+ cmp::max(1, cmp::min(log_value, self.max_edit_distance))
+ }
+
+ fn dist(&self) -> LevenshteinDistance {
+ LevenshteinDistance::new(self.max_edit_distance)
+ }
+}
+
+#[cfg(test)]
+pub struct MaxEdit {
+ max_edit_distance: usize,
+}
+
+#[cfg(test)]
+impl MaxEdit {
+ pub fn new(max_edit_distance: usize) -> Self {
+ Self { max_edit_distance }
+ }
+}
+
+#[cfg(test)]
+impl EditStrategy for MaxEdit {
+ fn distance_for_string(&self, _: &str) -> usize {
+ self.max_edit_distance
+ }
+
+ fn dist(&self) -> LevenshteinDistance {
+ LevenshteinDistance::new(self.max_edit_distance)
+ }
+}
+
+pub struct DictionaryResult {
+ pub prob: f64,
+ pub correction: String,
+}
+
+impl Hash for DictionaryResult {
+ fn hash<H>(&self, state: &mut H)
+ where
+ H: Hasher,
+ {
+ self.correction.hash(state);
+ }
+}
+
+impl PartialEq for DictionaryResult {
+ fn eq(&self, other: &Self) -> bool {
+ self.correction == other.correction
+ }
+}
+
+impl Eq for DictionaryResult {}
+
+#[derive(Error, Debug)]
+pub enum DictionaryError {
+ #[error("Underlying error from FST")]
+ Fst(#[from] fst::Error),
+
+ #[error("An IO error has occured")]
+ Io(#[from] io::Error),
+}
+
+pub type Result<T> = std::result::Result<T, DictionaryError>;
+
+enum InnerMap {
+ Memory(Map<Vec<u8>>),
+ File(Map<Mmap>),
+}
+
+impl InnerMap {
+ fn total_freq(&self) -> u64 {
+ let mut stream = match self {
+ InnerMap::Memory(map) => map.into_stream(),
+ InnerMap::File(map) => map.into_stream(),
+ };
+
+ let mut total_freq = 0;
+ while let Some((_, freq)) = stream.next() {
+ total_freq += freq;
+ }
+
+ total_freq
+ }
+}
+
+/// Dictionary that contains term frequency information
+pub struct Dictionary<const TOP_N: usize> {
+ cache: BTreeMap<String, u64>,
+ map: InnerMap,
+ folder_path: Option<String>,
+ total_freq: u64,
+}
+
+#[cfg(test)]
+impl Default for Dictionary<1_000> {
+ fn default() -> Self {
+ Dictionary::open::<&str>(None).unwrap()
+ }
+}
+
+impl<const TOP_N: usize> Dictionary<TOP_N> {
+ pub fn open<P: AsRef<Path>>(folder_path: Option<P>) -> Result<Self> {
+ if let Some(path) = folder_path.as_ref() {
+ fs::create_dir_all(path)?;
+ }
+
+ let folder_path = folder_path.map(|path| path.as_ref().to_str().unwrap().to_string());
+
+ let map = match &folder_path {
+ Some(path) => {
+ let dictionary_path = Path::new(path).join("dictionary");
+
+ if !dictionary_path.exists() {
+ let file = File::create(dictionary_path.clone())?;
+ let builder = MapBuilder::new(io::BufWriter::new(file))?;
+ builder.finish()?;
+ }
+
+ let dictionary_file = OpenOptions::new().read(true).open(dictionary_path)?;
+
+ let mmap = unsafe { Mmap::map(&dictionary_file)? };
+ InnerMap::File(Map::new(mmap)?)
+ }
+ None => InnerMap::Memory(Map::default()),
+ };
+
+ Ok(Dictionary {
+ total_freq: map.total_freq(),
+ map,
+ folder_path,
+ cache: BTreeMap::default(),
+ })
+ }
+
+ fn store_union(&mut self, mut union: Union) -> Result<()> {
+ match &self.folder_path {
+ Some(path) => {
+ let path = Path::new(path);
+ let wrt = io::BufWriter::new(File::create(path.join("new_dictionary"))?);
+ let mut builder = MapBuilder::new(wrt)?;
+
+ let mut heap = BinaryHeap::with_capacity(TOP_N + 1);
+
+ while let Some((key, values)) = union.next() {
+ let val: u64 = values.iter().map(|idx_val| idx_val.value).sum();
+
+ heap.push((
+ std::cmp::Reverse(val),
+ String::from_utf8_lossy(key).to_string(),
+ ));
+
+ if heap.len() > TOP_N {
+ heap.pop();
+ }
+ }
+
+ let top_values: BTreeMap<_, _> =
+ heap.into_iter().map(|(val, key)| (key, val.0)).collect();
+
+ for (key, value) in top_values {
+ builder.insert(key, value)?;
+ }
+
+ let path = Path::new(path);
+ builder.finish()?;
+ std::fs::rename(path.join("new_dictionary"), path.join("dictionary"))?;
+ let mmap = unsafe { Mmap::map(&File::open(path.join("dictionary"))?)? };
+ self.map = InnerMap::File(Map::new(mmap)?);
+ }
+ None => {
+ let mut builder = MapBuilder::memory();
+
+ let mut heap = BinaryHeap::with_capacity(TOP_N + 1);
+
+ while let Some((key, values)) = union.next() {
+ let val: u64 = values.iter().map(|idx_val| idx_val.value).sum();
+
+ heap.push((
+ std::cmp::Reverse(val),
+ String::from_utf8_lossy(key).to_string(),
+ ));
+
+ if heap.len() > TOP_N {
+ heap.pop();
+ }
+ }
+
+ let top_values: BTreeMap<_, _> =
+ heap.into_iter().map(|(val, key)| (key, val.0)).collect();
+
+ for (key, value) in top_values {
+ builder.insert(key, value)?;
+ }
+
+ let bytes = builder.into_inner().unwrap();
+ self.map = InnerMap::Memory(Map::new(bytes)?);
+ }
+ };
+
+ Ok(())
+ }
+
+ pub fn commit(&mut self) -> Result<()> {
+ let cache = mem::take(&mut self.cache);
+ let cache_map = Map::from_iter(cache.into_iter())?;
+ let map = mem::replace(&mut self.map, InnerMap::Memory(Map::default()));
+
+ let union = match &map {
+ InnerMap::File(map) => map.op().add(&cache_map).union(),
+ InnerMap::Memory(map) => map.op().add(&cache_map).union(),
+ };
+ self.store_union(union)?;
+
+ self.total_freq = self.map.total_freq();
+
+ Ok(())
+ }
+
+ pub fn insert(&mut self, term: &str) {
+ self.cache
+ .entry(
+ term.chars()
+ .into_iter()
+ .map(|c| c.to_ascii_lowercase())
+ .filter(|c| !matches!(c, ',' | '.' | '\\' | '=' | '*' | '(' | ')'))
+ .collect(),
+ )
+ .or_insert(0)
+ .add_assign(1);
+ }
+
+ #[inline]
+ pub fn all_probabilities(
+ &self,
+ term: &str,
+ edit_distance: usize,
+ ) -> Vec<HashSet<DictionaryResult>> {
+ let searcher = fst::automaton::Levenshtein::new(term, edit_distance as u32);
+ let mut res = Vec::with_capacity(edit_distance + 1);
+
+ if searcher.is_err() {
+ return res;
+ }
+
+ for _ in 0..edit_distance + 1 {
+ res.push(HashSet::new());
+ }
+
+ let searcher = searcher.unwrap();
+
+ let mut matches = self.perform_search(&searcher);
+ let dist_metric = LevenshteinDistance::new(edit_distance);
+
+ while let Some((correction, freq)) = matches.next() {
+ let correction = std::str::from_utf8(correction).unwrap().to_owned();
+ let dist = dist_metric.compare(term, &correction);
+
+ let prob = freq as f64 / self.total_freq as f64;
+
+ res[dist].insert(DictionaryResult { correction, prob });
+ }
+
+ res
+ }
+
+ fn perform_search<A: Automaton>(&self, searcher: A) -> fst::map::Stream<'_, A> {
+ match &self.map {
+ InnerMap::File(map) => map.search(searcher).into_stream(),
+ InnerMap::Memory(map) => map.search(searcher).into_stream(),
+ }
+ }
+
+ #[inline]
+ pub fn probability(&self, term: &str) -> Option<f64> {
+ let searcher = fst::automaton::Str::new(term);
+ let mut matches = self.perform_search(&searcher);
+
+ matches
+ .next()
+ .map(|(_key, frequency)| frequency as f64 / self.total_freq as f64)
+ }
+
+ #[cfg(test)]
+ pub fn contains(&self, term: &str) -> bool {
+ self.probability(term).is_some()
+ }
+
+ pub fn insert_page(&mut self, webpage: &Webpage) {
+ for text in webpage.html.clean_text().unwrap_or_default().split(' ') {
+ self.insert(text);
+ }
+ }
+
+ pub fn merge(mut self, mut other: Dictionary<TOP_N>) -> Self {
+ self.commit().unwrap();
+ other.commit().unwrap();
+
+ let map = mem::replace(&mut self.map, InnerMap::Memory(Map::default()));
+
+ let union = match (&map, &other.map) {
+ (InnerMap::Memory(map), InnerMap::Memory(other_map)) => map.op().add(other_map).union(),
+ (InnerMap::Memory(map), InnerMap::File(other_map)) => map.op().add(other_map).union(),
+ (InnerMap::File(map), InnerMap::Memory(other_map)) => map.op().add(other_map).union(),
+ (InnerMap::File(map), InnerMap::File(other_map)) => map.op().add(other_map).union(),
+ };
+
+ self.store_union(union).unwrap();
+ self.total_freq = self.map.total_freq();
+
+ self
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_probability() {
+ let mut dict = Dictionary::default();
+
+ dict.insert("this");
+ dict.insert("is");
+ dict.insert("a");
+ dict.insert("test");
+ dict.insert("test");
+
+ dict.commit().unwrap();
+
+ assert_eq!(dict.probability("this"), Some(0.2));
+ assert_eq!(dict.probability("is"), Some(0.2));
+ assert_eq!(dict.probability("a"), Some(0.2));
+ assert_eq!(dict.probability("test"), Some(0.4));
+ }
+
+ #[test]
+ fn test_uncontained_term() {
+ let mut dict = Dictionary::default();
+
+ dict.insert("this");
+ dict.insert("is");
+ dict.insert("a");
+ dict.insert("test");
+ dict.insert("test");
+
+ dict.commit().unwrap();
+
+ assert_eq!(dict.probability("the"), None);
+
+ assert!(dict.contains("this"));
+ assert!(dict.contains("is"));
+ assert!(dict.contains("test"));
+ assert!(!dict.contains("the"));
+ }
+
+ #[test]
+ fn test_probability_edit_3() {
+ let mut dict = Dictionary::default();
+
+ dict.insert("test");
+ dict.insert("twst");
+ dict.insert("kage");
+
+ dict.commit().unwrap();
+ let res = dict.all_probabilities("test", 2);
+
+ let mut num_elements = 0;
+
+ for set in res {
+ num_elements += set.len();
+ }
+
+ assert_eq!(num_elements, 2);
+ }
+
+ #[test]
+ fn merge() {
+ let mut dict1 = Dictionary::default();
+
+ dict1.insert("test");
+ dict1.insert("kage");
+
+ let mut dict2 = Dictionary::default();
+
+ dict2.insert("yay");
+ dict2.insert("ay");
+
+ let res = dict1.merge(dict2);
+
+ assert!(res.contains("test"));
+ assert!(res.contains("kage"));
+ assert!(res.contains("yay"));
+ assert!(res.contains("ay"));
+
+ assert_eq!(res.probability("test"), Some(1.0 / 4.0));
+ assert_eq!(res.probability("kage"), Some(1.0 / 4.0));
+ assert_eq!(res.probability("yay"), Some(1.0 / 4.0));
+ assert_eq!(res.probability("ay"), Some(1.0 / 4.0));
+ }
+
+ #[test]
+ fn only_stop_top_n() {
+ let mut dict: Dictionary<2> = Dictionary::open::<&str>(None).unwrap();
+
+ dict.insert("test");
+ dict.insert("test");
+ dict.insert("hej");
+ dict.insert("kage");
+ dict.insert("kage");
+
+ dict.commit().unwrap();
+
+ assert!(dict.contains("test"));
+ assert!(dict.contains("kage"));
+ assert!(!dict.contains("hej"));
+ }
+
+ #[test]
+ fn weird_characters_ignored() {
+ let mut dict = Dictionary::default();
+
+ dict.insert("lennon,");
+
+ dict.commit().unwrap();
+
+ assert!(dict.contains("lennon"));
+ assert!(!dict.contains("lennon,"));
+ }
+}
diff --git a/src/spell/distance.rs b/src/spell/distance.rs
new file mode 100644
index 00000000..515a0ae4
--- /dev/null
+++ b/src/spell/distance.rs
@@ -0,0 +1,108 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use serde::{Deserialize, Serialize};
+use std::hash::{Hash, Hasher};
+
+#[derive(Default, Serialize, Deserialize, Debug)]
+pub struct LevenshteinDistance {
+ dist: usize,
+}
+
+impl LevenshteinDistance {
+ pub fn new(distance: usize) -> Self {
+ LevenshteinDistance { dist: distance }
+ }
+
+ /// Function taken from https://github.com/febeling/edit-distance/blob/5597816456e7153cf69092f6ab5d0b4edb5e3797/src/lib.rs#L31
+ ///
+ /// The [Levenshtein edit distance][wikipedia] between two strings is
+ /// the number of individual single-character changes (insert, delete,
+ /// substitute) necessary to change string `a` into `b`.
+ ///
+ /// This can be a used to order search results, for fuzzy
+ /// auto-completion, and to find candidates for spelling correction.
+ ///
+ /// This function returns the edit distance between strings `a` and `b`.
+ ///
+ /// The runtime complexity is `O(m*n)`, where `m` and `n` are the
+ /// strings' lengths.
+ pub fn compare(&self, a: &str, b: &str) -> usize {
+ let len_a = a.chars().count();
+ let len_b = b.chars().count();
+ if len_a < len_b {
+ return self.compare(b, a);
+ }
+ // handle special case of 0 length
+ if len_a == 0 {
+ return len_b;
+ } else if len_b == 0 {
+ return len_a;
+ }
+
+ let len_b = len_b + 1;
+
+ let mut pre;
+ let mut tmp;
+ let mut cur = vec![0; len_b];
+
+ // initialize string
+ for (i, val) in cur.iter_mut().enumerate().take(len_b).skip(1) {
+ *val = i;
+ }
+
+ // calculate edit distance
+ for (i, ca) in a.chars().enumerate() {
+ // get first column for this row
+ pre = cur[0];
+ cur[0] = i + 1;
+ for (j, cb) in b.chars().enumerate() {
+ tmp = cur[j + 1];
+ cur[j + 1] = std::cmp::min(
+ // deletion
+ tmp + 1,
+ std::cmp::min(
+ // insertion
+ cur[j] + 1,
+ // match or substitution
+ pre + if ca == cb { 0 } else { 1 },
+ ),
+ );
+ pre = tmp;
+ }
+ }
+ cur[len_b - 1]
+ }
+}
+
+#[derive(Debug)]
+pub struct EditDistance {
+ pub dist: usize,
+ pub edited_string: String,
+}
+
+impl PartialEq for EditDistance {
+ fn eq(&self, other: &Self) -> bool {
+ self.edited_string == other.edited_string
+ }
+}
+
+impl Eq for EditDistance {}
+
+impl Hash for EditDistance {
+ fn hash<H: Hasher>(&self, state: &mut H) {
+ self.edited_string.hash(state);
+ }
+}
diff --git a/src/spell/mod.rs b/src/spell/mod.rs
new file mode 100644
index 00000000..3cdf2aec
--- /dev/null
+++ b/src/spell/mod.rs
@@ -0,0 +1,23 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+pub mod dictionary;
+pub mod distance;
+pub mod spell_checker;
+pub mod splitter;
+
+pub use self::dictionary::{Dictionary, DictionaryResult, EditStrategy, LogarithmicEdit};
+pub use self::spell_checker::SpellChecker;
+pub use self::splitter::TermSplitter;
diff --git a/src/spell/spell_checker.rs b/src/spell/spell_checker.rs
new file mode 100644
index 00000000..dfd55d78
--- /dev/null
+++ b/src/spell/spell_checker.rs
@@ -0,0 +1,170 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use crate::spell::dictionary::EditStrategy;
+use crate::spell::{Dictionary, DictionaryResult};
+use std::iter::FromIterator;
+
+pub struct SpellChecker<'a, T: EditStrategy, const DICT_N: usize> {
+ dict: &'a Dictionary<DICT_N>,
+ edit_strategy: T,
+}
+
+impl<'a, T: EditStrategy, const DICT_N: usize> SpellChecker<'a, T, DICT_N> {
+ pub fn new(dict: &'a Dictionary<DICT_N>, edit_strategy: T) -> Self {
+ SpellChecker {
+ dict,
+ edit_strategy,
+ }
+ }
+
+ pub fn correct_top(&self, term: &str, top_n: usize) -> Vec<String> {
+ let mut res = Vec::new();
+
+ // this is sorted by increasing edit distance
+ for corrections in self
+ .dict
+ .all_probabilities(term, self.edit_strategy.distance_for_string(term))
+ .iter_mut()
+ {
+ let mut corrections: Vec<&DictionaryResult> = Vec::from_iter(corrections.iter());
+ corrections.sort_by(|a, b| b.prob.partial_cmp(&a.prob).unwrap());
+
+ res.extend(
+ corrections
+ .into_iter()
+ .map(|dict_result| dict_result.correction.clone())
+ .take(top_n - res.len()),
+ );
+
+ if res.len() >= top_n {
+ return res;
+ }
+ }
+
+ res
+ }
+
+ pub fn correct(&self, term: &str) -> Option<String> {
+ if let Some(correction) = self.correct_top(term, 1).into_iter().next() {
+ if correction == term {
+ None
+ } else {
+ Some(correction)
+ }
+ } else {
+ None
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::spell::{dictionary::MaxEdit, LogarithmicEdit};
+
+ #[test]
+ fn simple_corrections() {
+ let mut dict = Dictionary::default();
+
+ dict.insert("this");
+ dict.insert("is");
+ dict.insert("a");
+ dict.insert("test");
+ dict.insert("test");
+ dict.insert("pest");
+
+ dict.commit().unwrap();
+
+ let spell_checker = SpellChecker::new(&dict, MaxEdit::new(1));
+
+ assert_eq!(spell_checker.correct("tst"), Some("test".to_string()));
+ assert_eq!(spell_checker.correct("ths"), Some("this".to_string()));
+ assert_eq!(spell_checker.correct("thes"), Some("this".to_string()));
+ assert_eq!(spell_checker.correct("is"), None);
+ assert_eq!(
+ spell_checker.correct_top("thes", 3),
+ vec!["this".to_string()]
+ );
+ assert_eq!(
+ spell_checker.correct_top("dest", 3),
+ vec!["test".to_string(), "pest".to_string()]
+ );
+ assert_eq!(
+ spell_checker.correct_top("dest", 1),
+ vec!["test".to_string()]
+ );
+ }
+
+ #[test]
+ fn correct_uncontained_word() {
+ let mut dict = Dictionary::default();
+
+ dict.insert("this");
+ dict.insert("is");
+ dict.insert("a");
+ dict.insert("test");
+ dict.insert("test");
+
+ dict.commit().unwrap();
+
+ let spell_checker = SpellChecker::new(&dict, MaxEdit::new(1));
+
+ assert_eq!(spell_checker.correct("what"), None);
+ assert!(spell_checker.correct_top("what", 2).is_empty());
+ }
+
+ #[test]
+ fn prioritise_low_distance_words() {
+ let mut dict = Dictionary::default();
+
+ dict.insert("this");
+ dict.insert("is");
+ dict.insert("a");
+ dict.insert("test");
+ dict.insert("contest");
+ dict.insert("contest");
+
+ dict.commit().unwrap();
+
+ let spell_checker = SpellChecker::new(&dict, MaxEdit::new(4));
+
+ assert_eq!(spell_checker.correct("dest"), Some("test".to_string()));
+ assert_eq!(
+ spell_checker.correct_top("dest", 3),
+ vec!["test".to_string(), "is".to_string(), "contest".to_string()]
+ );
+ }
+
+ #[test]
+ fn correct_sorting_multiple_hits() {
+ let mut dict = Dictionary::default();
+
+ dict.insert("the");
+ dict.insert("the");
+ dict.insert("the");
+ dict.insert("he");
+
+ dict.commit().unwrap();
+
+ let spell_checker = SpellChecker::new(&dict, LogarithmicEdit::new(4));
+
+ assert_eq!(spell_checker.correct("fhe"), Some("the".to_string()));
+ assert_eq!(
+ spell_checker.correct_top("fhe", 3),
+ vec!["the".to_string(), "he".to_string()]
+ );
+ }
+}
diff --git a/src/spell/splitter.rs b/src/spell/splitter.rs
new file mode 100644
index 00000000..876d098f
--- /dev/null
+++ b/src/spell/splitter.rs
@@ -0,0 +1,108 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use crate::spell::Dictionary;
+
+pub struct TermSplitter<'a, const DICT_N: usize> {
+ dict: &'a Dictionary<DICT_N>,
+}
+
+impl<'a, const DICT_N: usize> TermSplitter<'a, DICT_N> {
+ pub fn new(dict: &'a Dictionary<DICT_N>) -> Self {
+ TermSplitter { dict }
+ }
+ pub fn split(&self, text: &'a str) -> Vec<&'a str> {
+ let mut probs = vec![1.0];
+ let mut lasts: Vec<usize> = vec![0];
+
+ for i in 1..text.len() + 1 {
+ let mut best_prob_k = 0.0;
+ let mut best_index = 0;
+
+ for j in 0..i {
+ if !text.is_char_boundary(j) || !text.is_char_boundary(i) {
+ continue;
+ }
+
+ if let Some(prob) = &self.dict.probability(&text[j..i]) {
+ let new_prob = probs[j] * prob;
+ if new_prob > best_prob_k {
+ best_prob_k = new_prob;
+ best_index = j;
+ }
+ }
+ }
+
+ probs.push(best_prob_k);
+ lasts.push(best_index);
+ }
+
+ let mut words = Vec::new();
+ let mut i = text.len();
+
+ while i > 0 {
+ if i == text.len() && lasts[i] == 0 {
+ break;
+ }
+ words.push(&text[lasts[i]..i]);
+ i = lasts[i];
+ }
+
+ words.reverse();
+ words
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_split() {
+ let mut dictionary = Dictionary::default();
+
+ dictionary.insert("wicked");
+ dictionary.insert("weather");
+
+ dictionary.commit().unwrap();
+
+ assert_eq!(
+ TermSplitter::new(&dictionary).split("wickedweather"),
+ vec!["wicked", "weather"]
+ );
+
+ assert_eq!(TermSplitter::new(&dictionary).split("wicked").len(), 0);
+
+ assert_eq!(TermSplitter::new(&dictionary).split("udlæg").len(), 0);
+ }
+
+ #[test]
+ fn test_most_probable() {
+ let mut dictionary = Dictionary::default();
+
+ dictionary.insert("wicked");
+ dictionary.insert("wicked");
+ dictionary.insert("weather");
+ dictionary.insert("weather");
+ dictionary.insert("eat"); // "eat" is a substring of "weather"
+
+ dictionary.commit().unwrap();
+
+ assert_eq!(
+ TermSplitter::new(&dictionary).split("wickedweather"),
+ vec!["wicked", "weather"]
+ );
+ }
+}
diff --git a/templates/search.html b/templates/search.html
index 112fcc94..9144010e 100644
--- a/templates/search.html
+++ b/templates/search.html
@@ -15,45 +15,57 @@
</div>
<div class="flex pt-8">
- <div class="search-result flex flex-col space-y-3 pb-8 pl-40">
- {% for item in search_result %}
- <div class="flex max-w-3xl w-full p-3">
- <div class="flex flex-col w-full">
- <div class="text-sm mb-1 flex">
- <div class="flex items-center mr-2">
- <img src="/favicons/{{ item.domain }}" width="13px" />
+ <div class="flex flex-col pb-8 pl-40">
+ {% match spell_correction %}
+ {% when Some with (correction) %}
+ <div class="pl-3 pb-3 ">
+ Did you mean: <a class="italic title-link font-bold" href="/search?q={{ correction }}">{{ correction
+ }}</a>
+ </div>
+ {% when None %}
+ {% endmatch %}
+
+ <div class="search-result flex flex-col space-y-3">
+ {% for item in search_result %}
+ <div class="flex max-w-3xl w-full p-3">
+ <div class="flex flex-col w-full">
+ <div class="text-sm mb-1 flex">
+ <div class="flex items-center mr-2">
+ <img src="/favicons/{{ item.domain }}" width="13px" />
+ </div>
+ <div>
+ <a class="hover:no-underline text-gray-600" href="{{ item.url }}">{{ item.pretty_url
+ }}</a>
+ </div>
</div>
- <div>
- <a class="hover:no-underline text-gray-600" href="{{ item.url }}">{{ item.pretty_url }}</a>
+ <div class="text-lg font-medium mb-1">
+ <a class="title-link" href="{{ item.url }}">{{ item.title }}</a>
+ </div>
+ <div class="text-sm">
+ {% match item.last_updated %}
+ {% when Some with (val) %}
+ <span class="text-gray-500">
+ {{ val }}
+ </span> —
+ {% when None %}
+ {% endmatch %}
+ {{ item.snippet }}
</div>
</div>
- <div class="text-lg font-medium mb-1">
- <a class="title-link" href="{{ item.url }}">{{ item.title }}</a>
- </div>
- <div class="text-sm">
- {% match item.last_updated %}
- {% when Some with (val) %}
- <span class="text-gray-500">
- {{ val }}
- </span> —
- {% when None %}
- {% endmatch %}
- {{ item.snippet }}
- </div>
- </div>
- <div class="flex h-full items-center">
- <div class="h-20 w-20 pl-2 pt-1 pb-1">
- {% match item.primary_image_uuid %}
- {% when Some with (val) %}
- <a href="{{ item.url }}">
- <img class="h-full w-full object-cover rounded-full" src="/image/{{ val }}" />
- </a>
- {% when None %}
- {% endmatch %}
+ <div class="flex h-full items-center">
+ <div class="h-20 w-20 pl-2 pt-1 pb-1">
+ {% match item.primary_image_uuid %}
+ {% when Some with (val) %}
+ <a href="{{ item.url }}">
+ <img class="h-full w-full object-cover rounded-full" src="/image/{{ val }}" />
+ </a>
+ {% when None %}
+ {% endmatch %}
+ </div>
</div>
</div>
+ {% endfor %}
</div>
- {% endfor %}
</div>
{% match entity %}