Skip to content

web search done right

[WIP] Implement the importing of optics (#167)

Wesley Appler25c034457834
diff --git a/.cargo/config.toml b/.cargo/config.toml
deleted file mode 100644
index ddff4407..00000000
--- a/.cargo/config.toml
+++ /dev/null
@@ -1,2 +0,0 @@
-[build]
-rustflags = ["-C", "target-cpu=native"]
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 886c3b4d..e598e7d9 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -29,7 +29,7 @@ jobs:
- name: Checkout repository
uses: actions/checkout@v3
with:
- submodules: 'recursive'
+ submodules: "recursive"
- name: Install Rust
uses: dtolnay/rust-toolchain@stable
with:
@@ -37,8 +37,10 @@ jobs:
- uses: Swatinem/rust-cache@v2
- name: Install liburing
run: |
- sudo apt-get update
- sudo apt-get install -y liburing-dev
+ sudo apt-get update
+ sudo apt-get install -y liburing-dev
+ - name: Install wasm-pack
+ run: cargo install wasm-pack
- uses: taiki-e/install-action@v2
with:
tool: cargo-about
diff --git a/.vscode/settings.json b/.vscode/settings.json
index b3ae69bc..9e8c8b86 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -21,6 +21,7 @@
"rust-analyzer.linkedProjects": [
"Cargo.toml",
"./crates/optics-lsp/Cargo.toml",
+ "./crates/client-wasm/Cargo.toml",
"./crates/core/Cargo.toml"
]
}
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 7b68c7d7..3c07e48f 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -38,8 +38,9 @@ if they take our code without making theirs open source, which would breach the
* Install clang and npm
* Update ulimit. RocksDB tends to exceed the max number of allowed open files, so you will have to run `ulimit -n 10240` to increase the allowed max number of open files.
* Install [just](https://github.com/casey/just) by running `cargo install just`. This allows you to run the scripts in the `justfile` file. A justfile is basically a simple Makefile.
+* Install [wasm-pack](https://rustwasm.github.io/wasm-pack/installer/) by running `cargo install wasm-pack`. This will allow for generation and packaging of client side WebAssembly code.
* Run the command `just configure` which should automatically configures the rest of your dev environment. The script creates a python virtual environment, installs relevant dependencies, traces and exports the ML models and creates a small local index which you can use for development.
-* (Optional) Install [cargo-watch](https://github.com/watchexec/cargo-watch) by running `cargo install cargo-watch`. This makes frontend development easier.
+* Install [cargo-watch](https://github.com/watchexec/cargo-watch) by running `cargo install cargo-watch`.
* (Optional) Install [abeye](https://github.com/oeb25/abeye) by running `cargo install --git https://github.com/oeb25/abeye --locked`. This is used for generating the API client used by the frontend by running `just openapi`.
After the non optional steps you can now run `cargo test` and should see all tests passing. If you have installced `cargo-watch`, you should be able to run `just dev` to start the search server and launch the frontend at `0.0.0.0:8000`.
diff --git a/crates/client-wasm/.gitignore b/crates/client-wasm/.gitignore
new file mode 100644
index 00000000..b2bcbe85
--- /dev/null
+++ b/crates/client-wasm/.gitignore
@@ -0,0 +1 @@
+pkg/
\ No newline at end of file
diff --git a/crates/client-wasm/Cargo.lock b/crates/client-wasm/Cargo.lock
new file mode 100644
index 00000000..fad148cb
--- /dev/null
+++ b/crates/client-wasm/Cargo.lock
@@ -0,0 +1,909 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "aho-corasick"
+version = "1.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "ascii-canvas"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8824ecca2e851cec16968d54a01dd372ef8f95b244fb84b84e70128be347c3c6"
+dependencies = [
+ "term",
+]
+
+[[package]]
+name = "autocfg"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
+
+[[package]]
+name = "beef"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1"
+
+[[package]]
+name = "bit-set"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1"
+dependencies = [
+ "bit-vec",
+]
+
+[[package]]
+name = "bit-vec"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb"
+
+[[package]]
+name = "bitflags"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
+
+[[package]]
+name = "bitflags"
+version = "2.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf"
+
+[[package]]
+name = "bumpalo"
+version = "3.15.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ea184aa71bb362a1157c896979544cc23974e08fd265f29ea96b59f0b4a555b"
+
+[[package]]
+name = "cfg-if"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+
+[[package]]
+name = "client-wasm"
+version = "0.1.0"
+dependencies = [
+ "js-sys",
+ "optics",
+ "serde",
+ "serde-wasm-bindgen",
+ "serde_json",
+ "thiserror",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "crunchy"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"
+
+[[package]]
+name = "diff"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8"
+
+[[package]]
+name = "dirs-next"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1"
+dependencies = [
+ "cfg-if",
+ "dirs-sys-next",
+]
+
+[[package]]
+name = "dirs-sys-next"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d"
+dependencies = [
+ "libc",
+ "redox_users",
+ "winapi",
+]
+
+[[package]]
+name = "either"
+version = "1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a"
+
+[[package]]
+name = "ena"
+version = "0.14.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c533630cf40e9caa44bd91aadc88a75d75a4c3a12b4cfde353cbed41daa1e1f1"
+dependencies = [
+ "log",
+]
+
+[[package]]
+name = "equivalent"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
+
+[[package]]
+name = "fixedbitset"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
+
+[[package]]
+name = "fnv"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
+
+[[package]]
+name = "getrandom"
+version = "0.2.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi",
+]
+
+[[package]]
+name = "hashbrown"
+version = "0.14.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604"
+
+[[package]]
+name = "hermit-abi"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bd5256b483761cd23699d0da46cc6fd2ee3be420bbe6d020ae4a091e70b7e9fd"
+
+[[package]]
+name = "indexmap"
+version = "2.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "233cf39063f058ea2caae4091bf4a3ef70a653afbc026f5c4a4135d114e3c177"
+dependencies = [
+ "equivalent",
+ "hashbrown",
+ "serde",
+]
+
+[[package]]
+name = "is-terminal"
+version = "0.4.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b"
+dependencies = [
+ "hermit-abi",
+ "libc",
+ "windows-sys",
+]
+
+[[package]]
+name = "itertools"
+version = "0.10.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
+dependencies = [
+ "either",
+]
+
+[[package]]
+name = "itertools"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57"
+dependencies = [
+ "either",
+]
+
+[[package]]
+name = "itoa"
+version = "1.0.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c"
+
+[[package]]
+name = "js-sys"
+version = "0.3.68"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "406cda4b368d531c842222cf9d2600a9a4acce8d29423695379c6868a143a9ee"
+dependencies = [
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "lalrpop"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da4081d44f4611b66c6dd725e6de3169f9f63905421e8626fcb86b6a898998b8"
+dependencies = [
+ "ascii-canvas",
+ "bit-set",
+ "diff",
+ "ena",
+ "is-terminal",
+ "itertools 0.10.5",
+ "lalrpop-util",
+ "petgraph",
+ "pico-args",
+ "regex",
+ "regex-syntax 0.7.5",
+ "string_cache",
+ "term",
+ "tiny-keccak",
+ "unicode-xid",
+]
+
+[[package]]
+name = "lalrpop-util"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f35c735096c0293d313e8f2a641627472b83d01b937177fe76e5e2708d31e0d"
+dependencies = [
+ "regex",
+]
+
+[[package]]
+name = "libc"
+version = "0.2.153"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
+
+[[package]]
+name = "libredox"
+version = "0.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8"
+dependencies = [
+ "bitflags 2.4.2",
+ "libc",
+ "redox_syscall",
+]
+
+[[package]]
+name = "lock_api"
+version = "0.4.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45"
+dependencies = [
+ "autocfg",
+ "scopeguard",
+]
+
+[[package]]
+name = "log"
+version = "0.4.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
+
+[[package]]
+name = "logos"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c000ca4d908ff18ac99b93a062cb8958d331c3220719c52e77cb19cc6ac5d2c1"
+dependencies = [
+ "logos-derive",
+]
+
+[[package]]
+name = "logos-codegen"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc487311295e0002e452025d6b580b77bb17286de87b57138f3b5db711cded68"
+dependencies = [
+ "beef",
+ "fnv",
+ "proc-macro2",
+ "quote",
+ "regex-syntax 0.6.29",
+ "syn 2.0.50",
+]
+
+[[package]]
+name = "logos-derive"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dbfc0d229f1f42d790440136d941afd806bc9e949e2bcb8faa813b0f00d1267e"
+dependencies = [
+ "logos-codegen",
+]
+
+[[package]]
+name = "memchr"
+version = "2.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149"
+
+[[package]]
+name = "new_debug_unreachable"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54"
+
+[[package]]
+name = "once_cell"
+version = "1.19.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
+
+[[package]]
+name = "optics"
+version = "0.1.0"
+dependencies = [
+ "itertools 0.11.0",
+ "lalrpop",
+ "lalrpop-util",
+ "logos",
+ "once_cell",
+ "serde",
+ "thiserror",
+ "utoipa",
+]
+
+[[package]]
+name = "parking_lot"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
+dependencies = [
+ "lock_api",
+ "parking_lot_core",
+]
+
+[[package]]
+name = "parking_lot_core"
+version = "0.9.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "redox_syscall",
+ "smallvec",
+ "windows-targets 0.48.5",
+]
+
+[[package]]
+name = "petgraph"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9"
+dependencies = [
+ "fixedbitset",
+ "indexmap",
+]
+
+[[package]]
+name = "phf_shared"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096"
+dependencies = [
+ "siphasher",
+]
+
+[[package]]
+name = "pico-args"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5be167a7af36ee22fe3115051bc51f6e6c7054c9348e28deb4f49bd6f705a315"
+
+[[package]]
+name = "precomputed-hash"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
+
+[[package]]
+name = "proc-macro-error"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
+dependencies = [
+ "proc-macro-error-attr",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+ "version_check",
+]
+
+[[package]]
+name = "proc-macro-error-attr"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "version_check",
+]
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.78"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "redox_syscall"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa"
+dependencies = [
+ "bitflags 1.3.2",
+]
+
+[[package]]
+name = "redox_users"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4"
+dependencies = [
+ "getrandom",
+ "libredox",
+ "thiserror",
+]
+
+[[package]]
+name = "regex"
+version = "1.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-automata",
+ "regex-syntax 0.8.2",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax 0.8.2",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.6.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
+
+[[package]]
+name = "regex-syntax"
+version = "0.7.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da"
+
+[[package]]
+name = "regex-syntax"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"
+
+[[package]]
+name = "rustversion"
+version = "1.0.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4"
+
+[[package]]
+name = "ryu"
+version = "1.0.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1"
+
+[[package]]
+name = "scopeguard"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+
+[[package]]
+name = "serde"
+version = "1.0.197"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde-wasm-bindgen"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4c1432112bce8b966497ac46519535189a3250a3812cd27a999678a69756f79f"
+dependencies = [
+ "js-sys",
+ "serde",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.197"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.50",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.114"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0"
+dependencies = [
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "siphasher"
+version = "0.3.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d"
+
+[[package]]
+name = "smallvec"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7"
+
+[[package]]
+name = "string_cache"
+version = "0.8.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b"
+dependencies = [
+ "new_debug_unreachable",
+ "once_cell",
+ "parking_lot",
+ "phf_shared",
+ "precomputed-hash",
+]
+
+[[package]]
+name = "syn"
+version = "1.0.109"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
+dependencies = [
+ "proc-macro2",
+ "unicode-ident",
+]
+
+[[package]]
+name = "syn"
+version = "2.0.50"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "74f1bdc9872430ce9b75da68329d1c1746faf50ffac5f19e02b71e37ff881ffb"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "term"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f"
+dependencies = [
+ "dirs-next",
+ "rustversion",
+ "winapi",
+]
+
+[[package]]
+name = "thiserror"
+version = "1.0.57"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e45bcbe8ed29775f228095caf2cd67af7a4ccf756ebff23a306bf3e8b47b24b"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.57"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.50",
+]
+
+[[package]]
+name = "tiny-keccak"
+version = "2.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237"
+dependencies = [
+ "crunchy",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
+
+[[package]]
+name = "unicode-xid"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c"
+
+[[package]]
+name = "utoipa"
+version = "4.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "272ebdfbc99111033031d2f10e018836056e4d2c8e2acda76450ec7974269fa7"
+dependencies = [
+ "indexmap",
+ "serde",
+ "serde_json",
+ "utoipa-gen",
+]
+
+[[package]]
+name = "utoipa-gen"
+version = "4.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3c9f4d08338c1bfa70dde39412a040a884c6f318b3d09aaaf3437a1e52027fc"
+dependencies = [
+ "proc-macro-error",
+ "proc-macro2",
+ "quote",
+ "regex",
+ "syn 2.0.50",
+]
+
+[[package]]
+name = "version_check"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
+
+[[package]]
+name = "wasi"
+version = "0.11.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
+
+[[package]]
+name = "wasm-bindgen"
+version = "0.2.91"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c1e124130aee3fb58c5bdd6b639a0509486b0338acaaae0c84a5124b0f588b7f"
+dependencies = [
+ "cfg-if",
+ "wasm-bindgen-macro",
+]
+
+[[package]]
+name = "wasm-bindgen-backend"
+version = "0.2.91"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c9e7e1900c352b609c8488ad12639a311045f40a35491fb69ba8c12f758af70b"
+dependencies = [
+ "bumpalo",
+ "log",
+ "once_cell",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.50",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-macro"
+version = "0.2.91"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b30af9e2d358182b5c7449424f017eba305ed32a7010509ede96cdc4696c46ed"
+dependencies = [
+ "quote",
+ "wasm-bindgen-macro-support",
+]
+
+[[package]]
+name = "wasm-bindgen-macro-support"
+version = "0.2.91"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "642f325be6301eb8107a83d12a8ac6c1e1c54345a7ef1a9261962dfefda09e66"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.50",
+ "wasm-bindgen-backend",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-shared"
+version = "0.2.91"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4f186bd2dcf04330886ce82d6f33dd75a7bfcf69ecf5763b89fcde53b6ac9838"
+
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+
+[[package]]
+name = "windows-sys"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
+dependencies = [
+ "windows-targets 0.52.3",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
+dependencies = [
+ "windows_aarch64_gnullvm 0.48.5",
+ "windows_aarch64_msvc 0.48.5",
+ "windows_i686_gnu 0.48.5",
+ "windows_i686_msvc 0.48.5",
+ "windows_x86_64_gnu 0.48.5",
+ "windows_x86_64_gnullvm 0.48.5",
+ "windows_x86_64_msvc 0.48.5",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.52.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d380ba1dc7187569a8a9e91ed34b8ccfc33123bbacb8c0aed2d1ad7f3ef2dc5f"
+dependencies = [
+ "windows_aarch64_gnullvm 0.52.3",
+ "windows_aarch64_msvc 0.52.3",
+ "windows_i686_gnu 0.52.3",
+ "windows_i686_msvc 0.52.3",
+ "windows_x86_64_gnu 0.52.3",
+ "windows_x86_64_gnullvm 0.52.3",
+ "windows_x86_64_msvc 0.52.3",
+]
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.52.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68e5dcfb9413f53afd9c8f86e56a7b4d86d9a2fa26090ea2dc9e40fba56c6ec6"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.52.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8dab469ebbc45798319e69eebf92308e541ce46760b49b18c6b3fe5e8965b30f"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.52.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2a4e9b6a7cac734a8b4138a4e1044eac3404d8326b6c0f939276560687a033fb"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.52.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "28b0ec9c422ca95ff34a78755cfa6ad4a51371da2a5ace67500cf7ca5f232c58"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.52.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "704131571ba93e89d7cd43482277d6632589b18ecf4468f591fbae0a8b101614"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.52.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42079295511643151e98d61c38c0acc444e52dd42ab456f7ccfd5152e8ecf21c"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.52.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0770833d60a970638e989b3fa9fd2bb1aaadcf88963d1659fd7d9990196ed2d6"
diff --git a/crates/client-wasm/Cargo.toml b/crates/client-wasm/Cargo.toml
new file mode 100644
index 00000000..4ad33e9d
--- /dev/null
+++ b/crates/client-wasm/Cargo.toml
@@ -0,0 +1,19 @@
+[package]
+name = "client-wasm"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+crate-type = ["cdylib", "rlib"]
+
+[dependencies]
+js-sys = "0.3.60"
+serde = { version = "1.0.137", features = ["rc", "derive"] }
+serde_json = "1.0"
+serde-wasm-bindgen = "0.6.3"
+thiserror = "1.0.31"
+wasm-bindgen = "0.2.83"
+
+optics = { path = "../optics" }
+
+[workspace]
diff --git a/crates/client-wasm/src/lib.rs b/crates/client-wasm/src/lib.rs
new file mode 100644
index 00000000..b41c7f48
--- /dev/null
+++ b/crates/client-wasm/src/lib.rs
@@ -0,0 +1,68 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! WASM Bindings for Client Side JavaScript
+//!
+//! To be packaged with wasm-pack + vite and served to the browser
+
+use thiserror::Error;
+use wasm_bindgen::prelude::*;
+
+#[derive(Error, Debug)]
+pub enum Error {
+ #[error("Failed to serialize")]
+ Serialization(#[from] serde_wasm_bindgen::Error),
+
+ #[error("Optics error: {0}")]
+ OpticParse(#[from] optics::Error),
+
+ #[error("Json serialization error: {0}")]
+ SerdeJson(#[from] serde_json::Error),
+}
+
+impl From<Error> for JsValue {
+ fn from(val: Error) -> Self {
+ JsValue::from_str(&format!("{val:?}"))
+ }
+}
+
+#[wasm_bindgen]
+extern "C" {
+ #[wasm_bindgen(js_namespace = console, js_name = log)]
+ fn console_log(s: &str);
+}
+
+/// WASM Bindings for Optics
+///
+/// Used to prevent recreation of the parsing methods in JS.
+#[wasm_bindgen]
+pub struct Optic;
+
+#[wasm_bindgen]
+impl Optic {
+ /// Takes the contents of a .optic file and converts it to a Result containing either an error or a JSON serialized [`HostRankings`]
+ #[wasm_bindgen(js_name = parsePreferenceOptic)]
+ pub fn parse_preference_optic(contents: JsValue) -> Result<JsValue, Error> {
+ let optic_contents: String = serde_wasm_bindgen::from_value(contents)?;
+ let host_rankings = optics::Optic::parse(&optic_contents)?.host_rankings;
+
+ let rankings_json = serde_json::to_string(&host_rankings)?;
+
+ console_log(&("Parsed rankings to JSON: ".to_owned() + &rankings_json));
+
+ Ok(serde_wasm_bindgen::to_value(&rankings_json)?)
+ }
+}
diff --git a/crates/core/src/api/hosts.rs b/crates/core/src/api/hosts.rs
index cf54d18c..4e53d9df 100644
--- a/crates/core/src/api/hosts.rs
+++ b/crates/core/src/api/hosts.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use axum::{extract, Json};
+use axum::{body::Body, extract, response::{IntoResponse, Response}};
use http::StatusCode;
use optics::{HostRankings, Optic};
use utoipa::ToSchema;
@@ -34,11 +34,11 @@ pub struct HostsExportOpticParams {
)]
pub async fn hosts_export_optic(
extract::Json(HostsExportOpticParams { host_rankings }): extract::Json<HostsExportOpticParams>,
-) -> Result<Json<String>, StatusCode> {
+) -> Result<Response<Body>, StatusCode> {
let optic = Optic {
host_rankings,
..Default::default()
};
- Ok(Json(optic.to_string()))
+ Ok(optic.to_string().into_response())
}
diff --git a/crates/core/src/ranking/mod.rs b/crates/core/src/ranking/mod.rs
index aa0310f3..de5d75ac 100644
--- a/crates/core/src/ranking/mod.rs
+++ b/crates/core/src/ranking/mod.rs
@@ -334,6 +334,13 @@ mod tests {
.search(&SearchQuery {
query: "title".to_string(),
return_ranking_signals: true,
+ optic: Some(Optic {
+ rankings: vec![RankingCoeff {
+ target: RankingTarget::Signal("update_timestamp".to_string()),
+ value: 1_000_000.0,
+ }],
+ ..Default::default()
+ }),
..Default::default()
})
.expect("Search failed");
diff --git a/frontend/package.json b/frontend/package.json
index 15e12a47..0c93f8ee 100644
--- a/frontend/package.json
+++ b/frontend/package.json
@@ -2,9 +2,12 @@
"name": "frontend",
"version": "0.0.1",
"private": true,
+ "workspaces": [
+ "../crates/client-wasm/pkg"
+ ],
"scripts": {
"dev": "vite dev --port 8000",
- "build": "vite build",
+ "build": "npm run wasm && vite build",
"preview": "vite preview",
"test": "npm run test:integration && npm run test:unit",
"check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
@@ -49,7 +52,9 @@
"tslib": "^2.6.2",
"typescript": "^5.3.3",
"unplugin-icons": "^0.18.1",
- "vite": "^5.0.10",
+ "vite": "^5.1.4",
+ "vite-plugin-top-level-await": "^1.4.1",
+ "vite-plugin-wasm": "^3.3.0",
"vitest": "^1.0.4"
},
"type": "module",
diff --git a/frontend/src/lib/components/Button.svelte b/frontend/src/lib/components/Button.svelte
index dd6260db..6076b509 100644
--- a/frontend/src/lib/components/Button.svelte
+++ b/frontend/src/lib/components/Button.svelte
@@ -1,15 +1,6 @@
<script lang="ts">
import { twMerge } from 'tailwind-merge';
-
- type Kind =
- | 'primary'
- | 'seconday'
- | 'accent'
- | 'neutral'
- | 'info'
- | 'success'
- | 'warning'
- | 'error';
+ import { getButtonTailwindStyle, type ButtonKind } from '$lib/themes';
export let title: string | undefined = void 0;
export let form: string | undefined = void 0;
@@ -17,74 +8,10 @@
export let pale: boolean = false;
export let padding: boolean = true;
- export let kind: Kind = 'primary';
+ export let kind: ButtonKind = 'primary';
export let _class: string = '';
</script>
-<button
- on:click
- {title}
- {form}
- {type}
- class={twMerge(
- 'rounded-full py-2 transition active:scale-[98%]',
- padding ? 'px-4' : 'px-2',
- 'border border-transparent',
- pale
- ? [
- kind == 'primary' && [
- 'text-primary hover:text-primary-focus',
- 'bg-transparent hover:bg-primary/5 active:bg-primary/20',
- 'border-primary/20 hover:border-primary active:border-primary',
- ],
- kind == 'seconday' && [
- 'text-secondary hover:text-secondary-focus',
- 'bg-transparent hover:bg-secondary/5 active:bg-secondary/20',
- 'border-secondary/20 hover:border-secondary active:border-secondary',
- ],
- kind == 'accent' && [
- 'text-accent hover:text-accent-focus',
- 'bg-transparent hover:bg-accent/5 active:bg-accent/20',
- 'border-accent/20 hover:border-accent active:border-accent',
- ],
- kind == 'neutral' && [
- 'text-neutral hover:text-neutral-focus',
- 'bg-transparent hover:bg-neutral/5 active:bg-neutral/20',
- 'border-neutral/20 hover:border-neutral active:border-neutral',
- ],
- kind == 'info' && [
- 'text-info hover:text-info-focus',
- 'bg-transparent hover:bg-info/5 active:bg-info/20',
- 'border-info/20 hover:border-info active:border-info',
- ],
- kind == 'success' && [
- 'text-success hover:text-success-focus',
- 'bg-transparent hover:bg-success/5 active:bg-success/20',
- 'border-success/20 hover:border-success active:border-success',
- ],
- kind == 'warning' && [
- 'text-warning hover:text-warning-focus',
- 'bg-transparent hover:bg-warning/5 active:bg-warning/20',
- 'border-warning/20 hover:border-warning active:border-warning',
- ],
- kind == 'error' && [
- 'text-error hover:text-error-focus',
- 'bg-transparent hover:bg-error/5 active:bg-error/20',
- 'border-error/20 hover:border-error active:border-error',
- ],
- ]
- : [
- kind == 'primary' && 'bg-primary text-primary-content hover:bg-primary-focus',
- kind == 'seconday' && 'bg-secondary text-secondary-content hover:bg-secondary-focus',
- kind == 'accent' && 'bg-accent text-accent-content hover:bg-accent-focus',
- kind == 'neutral' && 'bg-neutral text-neutral-content hover:bg-neutral-focus',
- kind == 'info' && 'bg-info text-info-content hover:bg-info-focus',
- kind == 'success' && 'bg-success text-success-content hover:bg-success-focus',
- kind == 'warning' && 'bg-warning text-warning-content hover:bg-warning-focus',
- kind == 'error' && 'bg-error text-error-content hover:bg-error-focus',
- ],
- _class,
- )}
->
+<button on:click {title} {form} {type} class={getButtonTailwindStyle(padding, pale, kind, _class)}>
<slot />
</button>
diff --git a/frontend/src/lib/rankings.ts b/frontend/src/lib/rankings.ts
index d4dbd3a2..54b98e49 100644
--- a/frontend/src/lib/rankings.ts
+++ b/frontend/src/lib/rankings.ts
@@ -1,10 +1,15 @@
import LZString from 'lz-string';
-export type Ranking = 'liked' | 'disliked' | 'blocked';
-export type SiteRakings = Record<string, Ranking | undefined>;
+export enum Ranking {
+ LIKED = 'liked',
+ DISLIKED = 'disliked',
+ BLOCKED = 'blocked',
+}
+
+export type SiteRankings = Record<string, Ranking | undefined>;
export type RankedSites = Record<Ranking, string[]>;
-export const rankingsToRanked = (rankings: SiteRakings): RankedSites => {
+export const rankingsToRanked = (rankings: SiteRankings): RankedSites => {
const result: RankedSites = {
liked: [],
disliked: [],
@@ -17,10 +22,11 @@ export const rankingsToRanked = (rankings: SiteRakings): RankedSites => {
return result;
};
-export const rankedToRankings = (ranked: RankedSites): SiteRakings => {
- const result: SiteRakings = {};
- for (const ranking of ['liked', 'disliked', 'blocked'] as const) {
+export const rankedToRankings = (ranked: RankedSites): SiteRankings => {
+ const result: SiteRankings = {};
+
+ for (const [_, ranking] of Object.entries(Ranking)) {
for (const site of ranked[ranking]) {
result[site] = ranking;
}
diff --git a/frontend/src/lib/stores.ts b/frontend/src/lib/stores.ts
index 8cf068e7..d0fc2eac 100644
--- a/frontend/src/lib/stores.ts
+++ b/frontend/src/lib/stores.ts
@@ -1,7 +1,7 @@
import { browser } from '$app/environment';
import { writable } from 'svelte/store';
import type { OpticOption } from './optics';
-import type { SiteRakings } from './rankings';
+import type { SiteRankings } from './rankings';
import { api, type DisplayedWebpage } from './api';
import { match } from 'ts-pattern';
@@ -62,7 +62,7 @@ const ALLOW_STATS_KEY = 'allowStats';
export const allowStatsStore = writableLocalStorage<boolean>(ALLOW_STATS_KEY, true);
const HOST_RANKINGS_KEY = 'host_rankings';
-export const hostRankingsStore = writableLocalStorage<SiteRakings>(HOST_RANKINGS_KEY, {});
+export const hostRankingsStore = writableLocalStorage<SiteRankings>(HOST_RANKINGS_KEY, {});
const SEARCH_QUERY_KEY = 'searchQuery';
export const searchQueryStore = writableLocalStorage<string | undefined>(SEARCH_QUERY_KEY, void 0);
diff --git a/frontend/src/lib/themes.ts b/frontend/src/lib/themes.ts
index 93fe78f6..5cd3d5f2 100644
--- a/frontend/src/lib/themes.ts
+++ b/frontend/src/lib/themes.ts
@@ -1,3 +1,5 @@
+import { twMerge } from 'tailwind-merge';
+
const theme = (name: string, c: string) => ({ name, class: c });
export const THEMES = [
@@ -10,3 +12,85 @@ export const THEMES = [
theme('Nord', 'theme-nord'),
theme('Ayu Mirage', 'theme-ayu-mirage'),
];
+
+/*
+ Button Styling
+ Extracted here to be able to reuse this for labels and other elements.
+*/
+
+export type ButtonKind =
+ | 'primary'
+ | 'seconday'
+ | 'accent'
+ | 'neutral'
+ | 'info'
+ | 'success'
+ | 'warning'
+ | 'error';
+
+export const getButtonTailwindStyle = (
+ padding: boolean,
+ pale: boolean,
+ kind: ButtonKind,
+ _class: string,
+) => {
+ return twMerge(
+ 'rounded-full py-2 transition active:scale-[98%]',
+ padding ? 'px-4' : 'px-2',
+ 'border border-transparent',
+ pale
+ ? [
+ kind == 'primary' && [
+ 'text-primary hover:text-primary-focus',
+ 'bg-transparent hover:bg-primary/5 active:bg-primary/20',
+ 'border-primary/20 hover:border-primary active:border-primary',
+ ],
+ kind == 'seconday' && [
+ 'text-secondary hover:text-secondary-focus',
+ 'bg-transparent hover:bg-secondary/5 active:bg-secondary/20',
+ 'border-secondary/20 hover:border-secondary active:border-secondary',
+ ],
+ kind == 'accent' && [
+ 'text-accent hover:text-accent-focus',
+ 'bg-transparent hover:bg-accent/5 active:bg-accent/20',
+ 'border-accent/20 hover:border-accent active:border-accent',
+ ],
+ kind == 'neutral' && [
+ 'text-neutral hover:text-neutral-focus',
+ 'bg-transparent hover:bg-neutral/5 active:bg-neutral/20',
+ 'border-neutral/20 hover:border-neutral active:border-neutral',
+ ],
+ kind == 'info' && [
+ 'text-info hover:text-info-focus',
+ 'bg-transparent hover:bg-info/5 active:bg-info/20',
+ 'border-info/20 hover:border-info active:border-info',
+ ],
+ kind == 'success' && [
+ 'text-success hover:text-success-focus',
+ 'bg-transparent hover:bg-success/5 active:bg-success/20',
+ 'border-success/20 hover:border-success active:border-success',
+ ],
+ kind == 'warning' && [
+ 'text-warning hover:text-warning-focus',
+ 'bg-transparent hover:bg-warning/5 active:bg-warning/20',
+ 'border-warning/20 hover:border-warning active:border-warning',
+ ],
+ kind == 'error' && [
+ 'text-error hover:text-error-focus',
+ 'bg-transparent hover:bg-error/5 active:bg-error/20',
+ 'border-error/20 hover:border-error active:border-error',
+ ],
+ ]
+ : [
+ kind == 'primary' && 'bg-primary text-primary-content hover:bg-primary-focus',
+ kind == 'seconday' && 'bg-secondary text-secondary-content hover:bg-secondary-focus',
+ kind == 'accent' && 'bg-accent text-accent-content hover:bg-accent-focus',
+ kind == 'neutral' && 'bg-neutral text-neutral-content hover:bg-neutral-focus',
+ kind == 'info' && 'bg-info text-info-content hover:bg-info-focus',
+ kind == 'success' && 'bg-success text-success-content hover:bg-success-focus',
+ kind == 'warning' && 'bg-warning text-warning-content hover:bg-warning-focus',
+ kind == 'error' && 'bg-error text-error-content hover:bg-error-focus',
+ ],
+ _class,
+ );
+};
diff --git a/frontend/src/routes/search/Modal.svelte b/frontend/src/routes/search/Modal.svelte
index b247a903..0a544ae4 100644
--- a/frontend/src/routes/search/Modal.svelte
+++ b/frontend/src/routes/search/Modal.svelte
@@ -1,5 +1,5 @@
<script lang="ts">
- import type { Ranking } from '$lib/rankings';
+ import { Ranking } from '$lib/rankings';
import { hostRankingsStore, summarize } from '$lib/stores';
import type { DisplayedWebpage } from '$lib/api';
import { twJoin } from 'tailwind-merge';
@@ -31,17 +31,17 @@
const rankingChoices = [
{
- ranking: 'liked',
+ ranking: Ranking.LIKED,
kind: 'success',
Icon: HandThumbUp,
},
{
- ranking: 'disliked',
+ ranking: Ranking.DISLIKED,
kind: 'warning',
Icon: HandThumbDown,
},
{
- ranking: 'blocked',
+ ranking: Ranking.BLOCKED,
kind: 'error',
Icon: NoSymbol,
},
diff --git a/frontend/src/routes/settings/sites/+page.svelte b/frontend/src/routes/settings/sites/+page.svelte
index 0eeb9eb4..c66388da 100644
--- a/frontend/src/routes/settings/sites/+page.svelte
+++ b/frontend/src/routes/settings/sites/+page.svelte
@@ -2,30 +2,38 @@
import { api } from '$lib/api';
import Button from '$lib/components/Button.svelte';
import Site from '$lib/components/Site.svelte';
- import { rankingsToRanked } from '$lib/rankings';
+ import { getButtonTailwindStyle } from '$lib/themes';
+ import { rankingsToRanked, type RankedSites, Ranking } from '$lib/rankings';
import { hostRankingsStore } from '$lib/stores';
import { flip } from 'svelte/animate';
import { derived } from 'svelte/store';
+ import init, { Optic } from 'client-wasm';
+ import { onMount } from 'svelte';
+
+ onMount(async () => {
+ // Initialize the wasm module
+ await init();
+ });
const sections = [
{
title: 'Liked Sites',
description:
'Sites that are similar to these sites receive a boost during search. Their results are more likely to appear in your search results.',
- section: 'liked',
+ section: Ranking.LIKED,
},
{
title: 'Disliked Sites',
description:
'Sites that are similar to these sites gets de-prioritized during search. Their results are less likely to appear in your search results.',
- section: 'disliked',
+ section: Ranking.DISLIKED,
},
{
title: 'Blocked Sites',
description:
"These are the sites you have blocked. They won't appear in any of your searches.",
- section: 'blocked',
+ section: Ranking.BLOCKED,
},
] as const;
@@ -36,6 +44,45 @@
const ranked = derived(hostRankingsStore, ($rankings) => rankingsToRanked($rankings));
+ // Called when the user selects an optic file for import
+ const importOpticFile = (e: Event) => {
+ // Get an array of the uploaded files
+ let files: File[] = [...((<HTMLInputElement>e.target)?.files ?? new FileList())];
+
+ // Iterate through all files, attempt to get the contents & parse the optic
+ files.forEach((file) => {
+ if (file) {
+ const reader = new FileReader();
+ reader.readAsText(file, 'UTF-8');
+
+ reader.onload = (readerEvent) => {
+ const content = readerEvent.target?.result ?? '';
+ try {
+ const extractedRankings: RankedSites = JSON.parse(
+ Optic.parsePreferenceOptic(content as string),
+ );
+ // Iterate through all sites in each Ranking and pass them to rankSite
+ for (const [_, ranking] of Object.entries(Ranking)) {
+ const sites = extractedRankings[ranking];
+ sites.forEach((site) => rankSite(site, ranking));
+ }
+ } catch {
+ console.error(
+ `Failed to import optic from "${file.name}", please check the formatting.`,
+ );
+ }
+ };
+ }
+ });
+ };
+
+ const rankSite = (site: string, ranking: Ranking) => {
+ hostRankingsStore?.update(($rankings) => ({
+ ...$rankings,
+ [site]: $rankings[site] == ranking ? void 0 : ranking,
+ }));
+ };
+
const unrankSite = (site: string) => () => {
hostRankingsStore.update(($rankings) => ({ ...$rankings, [site]: void 0 }));
};
@@ -74,5 +121,16 @@
{#each buttons as { text, clear }}
<Button on:click={clearAndExport({ clear })}>{text}</Button>
{/each}
+ <input
+ type="file"
+ accept=".optic"
+ id="optic-import"
+ multiple
+ on:change={importOpticFile}
+ hidden
+ />
+ <label for="optic-import" class={getButtonTailwindStyle(false, false, 'primary', '')}>
+ Import from optic
+ </label>
</div>
</div>
diff --git a/frontend/src/wasm-pack-plugin/index.ts b/frontend/src/wasm-pack-plugin/index.ts
new file mode 100644
index 00000000..2aa25063
--- /dev/null
+++ b/frontend/src/wasm-pack-plugin/index.ts
@@ -0,0 +1,32 @@
+import { normalizePath, type Plugin } from 'vite';
+import childProcess from 'child_process';
+import * as path from 'path';
+
+type WasmPackConfig = {
+ crates: string[];
+};
+export default function wasmPack(config: WasmPackConfig): Plugin {
+ const jobs: (() => void)[] = [];
+
+ return {
+ name: 'wasm-pack',
+ buildStart() {
+ for (const crate of config.crates) {
+ const lib = path.resolve(normalizePath(crate));
+ const job = childProcess.spawn('cargo-watch', ['-s', 'wasm-pack build --target web'], {
+ cwd: lib,
+ env: { ...process.env, RUST_LOG: 'none' },
+ stdio: 'inherit',
+ });
+ jobs.push(() => {
+ job.kill();
+ });
+ }
+ },
+ buildEnd: () => {
+ for (const job of jobs) {
+ job();
+ }
+ },
+ };
+}
diff --git a/frontend/svelte.config.js b/frontend/svelte.config.js
index 1423c6bc..bfcd4db1 100644
--- a/frontend/svelte.config.js
+++ b/frontend/svelte.config.js
@@ -28,7 +28,7 @@ const config = {
csp: {
directives: {
'default-src': ['self'],
- 'script-src': ['self'],
+ 'script-src': ['self', 'wasm-unsafe-eval'],
// NOTE: Disabled in order to fetch optic sources client-side
// 'connect-src': ["'self'", 'http://localhost:3000/'],
'connect-src': ['*'],
diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts
index 7ba819a4..ab769d10 100644
--- a/frontend/vite.config.ts
+++ b/frontend/vite.config.ts
@@ -1,10 +1,26 @@
import { sveltekit } from '@sveltejs/kit/vite';
import { defineConfig } from 'vitest/config';
import Icons from 'unplugin-icons/vite';
+import wasm from 'vite-plugin-wasm';
+import topLevelAwait from 'vite-plugin-top-level-await';
+import wasmPack from './src/wasm-pack-plugin';
export default defineConfig({
- plugins: [sveltekit(), Icons({ compiler: 'svelte' })],
+ plugins: [
+ wasm(),
+ wasmPack({
+ crates: ['../crates/client-wasm/'],
+ }),
+ topLevelAwait(),
+ sveltekit(),
+ Icons({ compiler: 'svelte' }),
+ ],
test: {
include: ['src/**/*.{test,spec}.{js,ts}'],
},
+ server: {
+ fs: {
+ allow: ['../crates/client-wasm/pkg/'],
+ },
+ },
});
diff --git a/justfile b/justfile
index 07d36239..d467b77b 100644
--- a/justfile
+++ b/justfile
@@ -7,13 +7,13 @@ export RUST_LOG := env_var_or_default("RUST_LOG", "info,stract=debug")
export STRACT_CARGO_ARGS := env_var_or_default("STRACT_CARGO_ARGS", "")
@dev-api:
- cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- api configs/api.toml"
+ cargo watch -i frontend -i crates/client-wasm -x "run $STRACT_CARGO_ARGS -- api configs/api.toml"
@dev-search-server:
- cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- search-server configs/search_server.toml"
+ cargo watch -i frontend -i crates/client-wasm -x "run $STRACT_CARGO_ARGS -- search-server configs/search_server.toml"
@dev-entity-search-server:
- cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- entity-search-server configs/entity_search_server.toml"
+ cargo watch -i frontend -i crates/client-wasm -x "run $STRACT_CARGO_ARGS -- entity-search-server configs/entity_search_server.toml"
@dev-webgraph:
- cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- webgraph server configs/webgraph/host_server.toml" && cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- webgraph server configs/webgraph/page_server.toml"
+ cargo watch -i frontend -i crates/client-wasm -x "run $STRACT_CARGO_ARGS -- webgraph server configs/webgraph/host_server.toml" && cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- webgraph server configs/webgraph/page_server.toml"
@dev-llm:
./.venv/bin/python3 -m llama_cpp.server --model data/mistral-7b-instruct-v0.2.Q4_K_M.gguf --port 4000
@dev-frontend:
diff --git a/scripts/ci/check b/scripts/ci/check
index f89b5936..0312e07a 100755
--- a/scripts/ci/check
+++ b/scripts/ci/check
@@ -5,4 +5,4 @@ cargo check
cargo check --no-default-features
# skip frontend check until https://github.com/sveltejs/kit/issues/11906 is fixed
-# cd frontend && npm install && npm run check
+# cd frontend && npm run wasm && npm install && npm run check