Skip to content

web search done right

move neural network models from torch to candle

Mikkel Denkerd7e564d91a21
diff --git a/.gitmodules b/.gitmodules
index 73941dd1..e9ef195d 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,6 +1,3 @@
[submodule "optics/testcases"]
path = optics/testcases/samples
url = https://github.com/StractOrg/sample-optics
-[submodule "crates/ggml-sys/ggml-src"]
- path = crates/ggml-sys/ggml-src
- url = https://github.com/ggerganov/ggml
diff --git a/Cargo.lock b/Cargo.lock
index 102a6d18..1fcd9f3d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -460,29 +460,6 @@ dependencies = [
"syn 2.0.38",
]
-[[package]]
-name = "bindgen"
-version = "0.69.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4c69fae65a523209d34240b60abe0c42d33d1045d445c0839d8a4894a736e2d"
-dependencies = [
- "bitflags 2.4.1",
- "cexpr",
- "clang-sys",
- "lazy_static",
- "lazycell",
- "log",
- "peeking_take_while",
- "prettyplease",
- "proc-macro2",
- "quote",
- "regex",
- "rustc-hash",
- "shlex",
- "syn 2.0.38",
- "which",
-]
-
[[package]]
name = "bit-set"
version = "0.5.3"
@@ -643,6 +620,61 @@ dependencies = [
"zip",
]
+[[package]]
+name = "candle-core"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6db8659ea87ee8197d2fc627348916cce0561330ee7ae3874e771691d3cecb2f"
+dependencies = [
+ "byteorder 1.5.0",
+ "gemm",
+ "half 2.3.1",
+ "memmap2",
+ "num-traits",
+ "num_cpus",
+ "rand 0.8.5",
+ "rand_distr",
+ "rayon",
+ "safetensors 0.4.2",
+ "thiserror",
+ "yoke",
+ "zip",
+]
+
+[[package]]
+name = "candle-nn"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ddce8312032760a6791d6adc9c56dc54fd7c1be38d85dcc4862f1c75228bbc7"
+dependencies = [
+ "candle-core",
+ "half 2.3.1",
+ "num-traits",
+ "rayon",
+ "safetensors 0.4.2",
+ "serde",
+ "thiserror",
+]
+
+[[package]]
+name = "candle-transformers"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68834a0cacb7e002d1f4abfe26a7cd1237e2ba342fddcf2e30913c4edb96409d"
+dependencies = [
+ "byteorder 1.5.0",
+ "candle-core",
+ "candle-nn",
+ "num-traits",
+ "rand 0.8.5",
+ "rayon",
+ "serde",
+ "serde_json",
+ "serde_plain",
+ "tracing",
+ "wav",
+]
+
[[package]]
name = "cast"
version = "0.3.0"
@@ -799,15 +831,6 @@ version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961"
-[[package]]
-name = "cmake"
-version = "0.1.50"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130"
-dependencies = [
- "cc",
-]
-
[[package]]
name = "color_quant"
version = "1.1.0"
@@ -1229,6 +1252,16 @@ dependencies = [
"dtoa",
]
+[[package]]
+name = "dyn-stack"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56e53799688f5632f364f8fb387488dd05db9fe45db7011be066fc20e7027f8b"
+dependencies = [
+ "bytemuck",
+ "reborrow",
+]
+
[[package]]
name = "either"
version = "1.9.0"
@@ -1259,6 +1292,18 @@ dependencies = [
"cfg-if",
]
+[[package]]
+name = "enum-as-inner"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5ffccbb6966c05b32ef8fbac435df276c4ae4d3dc55a8cd0eb9745e6c12f546a"
+dependencies = [
+ "heck 0.4.1",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.38",
+]
+
[[package]]
name = "equivalent"
version = "1.0.1"
@@ -1558,6 +1603,124 @@ dependencies = [
"byteorder 1.5.0",
]
+[[package]]
+name = "gemm"
+version = "0.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e97d506c68f4fb12325b52a638e7d54cc87e3593a4ded0de60218b6dfd65f645"
+dependencies = [
+ "dyn-stack",
+ "gemm-c32",
+ "gemm-c64",
+ "gemm-common",
+ "gemm-f16",
+ "gemm-f32",
+ "gemm-f64",
+ "num-complex",
+ "num-traits",
+ "paste",
+ "raw-cpuid",
+ "seq-macro",
+]
+
+[[package]]
+name = "gemm-c32"
+version = "0.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0dd16f26e8f34661edc906d8c9522b59ec1655c865a98a58950d0246eeaca9da"
+dependencies = [
+ "dyn-stack",
+ "gemm-common",
+ "num-complex",
+ "num-traits",
+ "paste",
+ "raw-cpuid",
+ "seq-macro",
+]
+
+[[package]]
+name = "gemm-c64"
+version = "0.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8e34381bc060b47fbd25522a281799ef763cd27f43bbd1783d935774659242a"
+dependencies = [
+ "dyn-stack",
+ "gemm-common",
+ "num-complex",
+ "num-traits",
+ "paste",
+ "raw-cpuid",
+ "seq-macro",
+]
+
+[[package]]
+name = "gemm-common"
+version = "0.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22518a76339b09276f77c3166c44262e55f633712fe8a44fd0573505887feeab"
+dependencies = [
+ "bytemuck",
+ "dyn-stack",
+ "half 2.3.1",
+ "num-complex",
+ "num-traits",
+ "once_cell",
+ "paste",
+ "pulp",
+ "raw-cpuid",
+ "rayon",
+ "seq-macro",
+ "sysctl",
+]
+
+[[package]]
+name = "gemm-f16"
+version = "0.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "70409bbf3ef83b38cbe4a58cd4b797c1c27902505bdd926a588ea61b6c550a84"
+dependencies = [
+ "dyn-stack",
+ "gemm-common",
+ "gemm-f32",
+ "half 2.3.1",
+ "num-complex",
+ "num-traits",
+ "paste",
+ "raw-cpuid",
+ "rayon",
+ "seq-macro",
+]
+
+[[package]]
+name = "gemm-f32"
+version = "0.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5ea3068edca27f100964157211782eba19e961aa4d0d2bdac3e1775a51aa7680"
+dependencies = [
+ "dyn-stack",
+ "gemm-common",
+ "num-complex",
+ "num-traits",
+ "paste",
+ "raw-cpuid",
+ "seq-macro",
+]
+
+[[package]]
+name = "gemm-f64"
+version = "0.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5fd41e8f5a60dce8d8acd852a3f4b22f8e18be957e1937731be692c037652510"
+dependencies = [
+ "dyn-stack",
+ "gemm-common",
+ "num-complex",
+ "num-traits",
+ "paste",
+ "raw-cpuid",
+ "seq-macro",
+]
+
[[package]]
name = "generator"
version = "0.7.5"
@@ -1605,22 +1768,6 @@ dependencies = [
"wasm-bindgen",
]
-[[package]]
-name = "ggml"
-version = "0.1.0"
-dependencies = [
- "ggml-sys",
-]
-
-[[package]]
-name = "ggml-sys"
-version = "0.1.0"
-dependencies = [
- "bindgen 0.69.2",
- "cc",
- "cmake",
-]
-
[[package]]
name = "ghash"
version = "0.5.0"
@@ -1703,8 +1850,12 @@ version = "2.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc52e53916c08643f1b56ec082790d1e86a32e58dc5268f897f313fbae7b4872"
dependencies = [
+ "bytemuck",
"cfg-if",
"crunchy",
+ "num-traits",
+ "rand 0.8.5",
+ "rand_distr",
"serde",
]
@@ -1770,15 +1921,6 @@ dependencies = [
"digest",
]
-[[package]]
-name = "home"
-version = "0.5.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5"
-dependencies = [
- "windows-sys 0.52.0",
-]
-
[[package]]
name = "html5ever"
version = "0.26.0"
@@ -2267,9 +2409,9 @@ checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25"
[[package]]
name = "libc"
-version = "0.2.149"
+version = "0.2.153"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b"
+checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
[[package]]
name = "libloading"
@@ -2293,7 +2435,7 @@ version = "0.11.0+8.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3386f101bcb4bd252d8e9d2fb41ec3b0862a15a62b478c355b2982efa469e3e"
dependencies = [
- "bindgen 0.65.1",
+ "bindgen",
"bzip2-sys",
"cc",
"glob",
@@ -2551,11 +2693,12 @@ dependencies = [
[[package]]
name = "memmap2"
-version = "0.9.0"
+version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "deaba38d7abf1d4cca21cc89e932e542ba2b9258664d2a9ef0e61512039c9375"
+checksum = "fe751422e4a8caa417e13c3ea66452215d7d63e19e604f4980461212f3ae1322"
dependencies = [
"libc",
+ "stable_deref_trait",
]
[[package]]
@@ -2741,6 +2884,7 @@ version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ba157ca0885411de85d6ca030ba7e2a83a28636056c7c699b07c8b6f7383214"
dependencies = [
+ "bytemuck",
"num-traits",
]
@@ -3363,6 +3507,18 @@ dependencies = [
"psl-types",
]
+[[package]]
+name = "pulp"
+version = "0.18.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "16785ee69419641c75affff7c9fdbdb7c0ab26dc9a5fb5218c2a2e9e4ef2087d"
+dependencies = [
+ "bytemuck",
+ "libm",
+ "num-complex",
+ "reborrow",
+]
+
[[package]]
name = "qoi"
version = "0.4.1"
@@ -3475,6 +3631,16 @@ dependencies = [
"getrandom 0.2.10",
]
+[[package]]
+name = "rand_distr"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31"
+dependencies = [
+ "num-traits",
+ "rand 0.8.5",
+]
+
[[package]]
name = "rand_hc"
version = "0.2.0"
@@ -3511,6 +3677,15 @@ dependencies = [
"rand_core 0.6.4",
]
+[[package]]
+name = "raw-cpuid"
+version = "10.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332"
+dependencies = [
+ "bitflags 1.3.2",
+]
+
[[package]]
name = "rawpointer"
version = "0.2.1"
@@ -3548,6 +3723,12 @@ dependencies = [
"crossbeam-utils",
]
+[[package]]
+name = "reborrow"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430"
+
[[package]]
name = "redox_syscall"
version = "0.2.16"
@@ -3667,6 +3848,12 @@ dependencies = [
"winreg",
]
+[[package]]
+name = "riff"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9b1a3d5f46d53f4a3478e2be4a5a5ce5108ea58b100dcd139830eae7f79a3a1"
+
[[package]]
name = "ring"
version = "0.17.4"
@@ -3888,6 +4075,16 @@ dependencies = [
"serde_json",
]
+[[package]]
+name = "safetensors"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8d980e6bfb34436fb0a81e42bc41af43f11805bbbca443e7f68e9faaabe669ed"
+dependencies = [
+ "serde",
+ "serde_json",
+]
+
[[package]]
name = "same-file"
version = "1.0.6"
@@ -4030,6 +4227,12 @@ version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090"
+[[package]]
+name = "seq-macro"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4"
+
[[package]]
name = "serde"
version = "1.0.189"
@@ -4071,6 +4274,15 @@ dependencies = [
"serde",
]
+[[package]]
+name = "serde_plain"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ce1fc6db65a611022b23a0dec6975d63fb80a302cb3388835ff02c097258d50"
+dependencies = [
+ "serde",
+]
+
[[package]]
name = "serde_spanned"
version = "0.6.3"
@@ -4286,6 +4498,9 @@ dependencies = [
"bytemuck",
"byteorder 1.5.0",
"bzip2",
+ "candle-core",
+ "candle-nn",
+ "candle-transformers",
"chitchat",
"chrono",
"clap",
@@ -4300,7 +4515,6 @@ dependencies = [
"fnv",
"fst",
"futures",
- "ggml",
"half 2.3.1",
"hashbrown 0.14.1",
"http 1.0.0",
@@ -4338,7 +4552,7 @@ dependencies = [
"rocksdb",
"rust-s3",
"rust-stemmers",
- "safetensors",
+ "safetensors 0.3.3",
"scylla",
"serde",
"serde_json",
@@ -4448,6 +4662,31 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
+[[package]]
+name = "synstructure"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.38",
+]
+
+[[package]]
+name = "sysctl"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec7dddc5f0fee506baf8b9fdb989e242f17e4b11c61dfbb0635b705217199eea"
+dependencies = [
+ "bitflags 2.4.1",
+ "byteorder 1.5.0",
+ "enum-as-inner",
+ "libc",
+ "thiserror",
+ "walkdir",
+]
+
[[package]]
name = "system-configuration"
version = "0.5.1"
@@ -4630,7 +4869,7 @@ dependencies = [
"libc",
"ndarray",
"rand 0.8.5",
- "safetensors",
+ "safetensors 0.3.3",
"thiserror",
"torch-sys",
"zip",
@@ -5401,6 +5640,15 @@ dependencies = [
"web-sys",
]
+[[package]]
+name = "wav"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a65e199c799848b4f997072aa4d673c034f80f40191f97fe2f0a23f410be1609"
+dependencies = [
+ "riff",
+]
+
[[package]]
name = "web-sys"
version = "0.3.64"
@@ -5428,18 +5676,6 @@ dependencies = [
"serde",
]
-[[package]]
-name = "which"
-version = "4.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7"
-dependencies = [
- "either",
- "home",
- "once_cell",
- "rustix",
-]
-
[[package]]
name = "winapi"
version = "0.3.9"
@@ -5498,15 +5734,6 @@ dependencies = [
"windows-targets 0.48.5",
]
-[[package]]
-name = "windows-sys"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
-dependencies = [
- "windows-targets 0.52.0",
-]
-
[[package]]
name = "windows-targets"
version = "0.42.2"
@@ -5537,21 +5764,6 @@ dependencies = [
"windows_x86_64_msvc 0.48.5",
]
-[[package]]
-name = "windows-targets"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd"
-dependencies = [
- "windows_aarch64_gnullvm 0.52.0",
- "windows_aarch64_msvc 0.52.0",
- "windows_i686_gnu 0.52.0",
- "windows_i686_msvc 0.52.0",
- "windows_x86_64_gnu 0.52.0",
- "windows_x86_64_gnullvm 0.52.0",
- "windows_x86_64_msvc 0.52.0",
-]
-
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.42.2"
@@ -5564,12 +5776,6 @@ version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
-[[package]]
-name = "windows_aarch64_gnullvm"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea"
-
[[package]]
name = "windows_aarch64_msvc"
version = "0.42.2"
@@ -5582,12 +5788,6 @@ version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
-[[package]]
-name = "windows_aarch64_msvc"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef"
-
[[package]]
name = "windows_i686_gnu"
version = "0.42.2"
@@ -5600,12 +5800,6 @@ version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
-[[package]]
-name = "windows_i686_gnu"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313"
-
[[package]]
name = "windows_i686_msvc"
version = "0.42.2"
@@ -5618,12 +5812,6 @@ version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
-[[package]]
-name = "windows_i686_msvc"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a"
-
[[package]]
name = "windows_x86_64_gnu"
version = "0.42.2"
@@ -5636,12 +5824,6 @@ version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
-[[package]]
-name = "windows_x86_64_gnu"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd"
-
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.42.2"
@@ -5654,12 +5836,6 @@ version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
-[[package]]
-name = "windows_x86_64_gnullvm"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e"
-
[[package]]
name = "windows_x86_64_msvc"
version = "0.42.2"
@@ -5672,12 +5848,6 @@ version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
-[[package]]
-name = "windows_x86_64_msvc"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04"
-
[[package]]
name = "winnow"
version = "0.5.17"
@@ -5724,6 +5894,51 @@ dependencies = [
"linked-hash-map",
]
+[[package]]
+name = "yoke"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "65e71b2e4f287f467794c671e2b8f8a5f3716b3c829079a1c44740148eff07e4"
+dependencies = [
+ "serde",
+ "stable_deref_trait",
+ "yoke-derive",
+ "zerofrom",
+]
+
+[[package]]
+name = "yoke-derive"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e6936f0cce458098a201c245a11bef556c6a0181129c7034d10d76d1ec3a2b8"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.38",
+ "synstructure",
+]
+
+[[package]]
+name = "zerofrom"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "655b0814c5c0b19ade497851070c640773304939a6c0fd5f5fb43da0696d05b7"
+dependencies = [
+ "zerofrom-derive",
+]
+
+[[package]]
+name = "zerofrom-derive"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6a647510471d372f2e6c2e6b7219e44d8c574d24fdc11c610a61455782f18c3"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.38",
+ "synstructure",
+]
+
[[package]]
name = "zimba"
version = "0.1.0"
diff --git a/Cargo.toml b/Cargo.toml
index 58b26b64..ccdda92b 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -4,7 +4,6 @@ members = [
"crates/optics",
"crates/kuchiki",
"crates/zimba",
- "crates/ggml-sys", "crates/ggml",
]
resolver = "2"
@@ -29,6 +28,9 @@ bitvec = "1.0.1"
bytemuck = {version = "1.13.1", features = ["derive"]}
byteorder = "1.4.3"
bzip2 = "0.4.4"
+candle-transformers = "0.3.3"
+candle-nn = "0.3.3"
+candle-core = "0.3.3"
chitchat = "0.5.0"
chrono = {version = "0.4.23", features = ["serde"]}
clap = {version = "4.4.6", features = ["derive"]}
diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml
index 173cc652..402db030 100644
--- a/crates/core/Cargo.toml
+++ b/crates/core/Cargo.toml
@@ -34,6 +34,9 @@ bitvec = {workspace = true}
bytemuck = {workspace = true}
byteorder = {workspace = true}
bzip2 = {workspace = true}
+candle-transformers = {workspace = true}
+candle-nn = {workspace = true}
+candle-core = {workspace = true}
chitchat = {workspace = true}
chrono = {workspace = true}
clap = {workspace = true}
@@ -47,7 +50,6 @@ flate2 = {workspace = true}
fnv = {workspace = true}
fst = {workspace = true}
futures = {workspace = true}
-ggml = {path = "../ggml"}
half = {workspace = true}
hashbrown = {workspace = true}
http = {workspace = true}
diff --git a/crates/core/src/llm_utils.rs b/crates/core/src/llm_utils.rs
index ea87c4bf..60dc04cc 100644
--- a/crates/core/src/llm_utils.rs
+++ b/crates/core/src/llm_utils.rs
@@ -19,20 +19,8 @@ use anyhow::anyhow;
use eventsource_stream::Eventsource;
use futures::stream::Stream;
#[cfg(feature = "libtorch")]
-use tch::Tensor;
use tokio_stream::StreamExt;
-#[cfg(feature = "libtorch")]
-pub struct ClonableTensor(pub Tensor);
-
-#[cfg(feature = "libtorch")]
-impl Clone for ClonableTensor {
- fn clone(&self) -> Self {
- let out = Tensor::empty(self.0.size(), (self.0.kind(), self.0.device()));
- ClonableTensor(self.0.clone(&out))
- }
-}
-
pub struct OpenAiApi {
api: String,
key: Option<String>,
diff --git a/crates/core/src/models/bert.rs b/crates/core/src/models/bert.rs
index d7733cf6..29077f79 100644
--- a/crates/core/src/models/bert.rs
+++ b/crates/core/src/models/bert.rs
@@ -1,44 +1,569 @@
-use ggml::{nn, Context, Tensor};
+//! this file has been copied from the candle-transformers project and modified
+//! to include an optional pooler layer and use an attention mask for padding tokens.
+//!
+//! The original file can be found at:
+//!
+//! https://github.com/huggingface/candle/blob/main/candle-examples/examples/bert/main.rs
-struct BertEmbeddings {
- word_embeddings: nn::Embedding,
- position_embeddings: nn::Embedding,
- token_type_embeddings: nn::Embedding,
- layer_norm: nn::LayerNorm,
+use candle_core::{Device, Result, Tensor};
+use candle_nn::{embedding, Embedding, Module, VarBuilder};
+use candle_transformers::models::with_tracing::{layer_norm, linear, LayerNorm, Linear};
+use serde::Deserialize;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum HiddenAct {
+ Gelu,
+ GeluApproximate,
+ Relu,
}
-impl BertEmbeddings {
- pub fn new(ctx: &mut Context, vocab_size: u64, hidden_size: u64) -> Self {
- let word_embeddings = nn::Embedding::new(ctx, vocab_size, hidden_size);
- let position_embeddings = nn::Embedding::new(ctx, 512, hidden_size);
- let token_type_embeddings = nn::Embedding::new(ctx, 2, hidden_size);
- let layer_norm = nn::LayerNorm::new(1e-12);
+struct HiddenActLayer {
+ act: HiddenAct,
+ span: tracing::Span,
+}
+
+impl HiddenActLayer {
+ fn new(act: HiddenAct) -> Self {
+ let span = tracing::span!(tracing::Level::TRACE, "hidden-act");
+ Self { act, span }
+ }
+
+ fn forward(&self, xs: &Tensor) -> candle_core::Result<Tensor> {
+ let _enter = self.span.enter();
+ match self.act {
+ // https://github.com/huggingface/transformers/blob/cd4584e3c809bb9e1392ccd3fe38b40daba5519a/src/transformers/activations.py#L213
+ HiddenAct::Gelu => xs.gelu_erf(),
+ HiddenAct::GeluApproximate => xs.gelu(),
+ HiddenAct::Relu => xs.relu(),
+ }
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Default)]
+#[serde(rename_all = "lowercase")]
+pub enum PositionEmbeddingType {
+ #[default]
+ Absolute,
+}
+// https://github.com/huggingface/transformers/blob/6eedfa6dd15dc1e22a55ae036f681914e5a0d9a1/src/transformers/models/bert/configuration_bert.py#L1
+#[derive(Debug, Clone, PartialEq, Deserialize)]
+pub struct Config {
+ pub vocab_size: usize,
+ pub hidden_size: usize,
+ pub num_hidden_layers: usize,
+ pub num_attention_heads: usize,
+ pub intermediate_size: usize,
+ pub hidden_act: HiddenAct,
+ pub hidden_dropout_prob: f64,
+ pub max_position_embeddings: usize,
+ pub type_vocab_size: usize,
+ pub initializer_range: f64,
+ pub layer_norm_eps: f64,
+ pub pad_token_id: usize,
+ #[serde(default)]
+ pub position_embedding_type: PositionEmbeddingType,
+ #[serde(default)]
+ pub use_cache: bool,
+ pub classifier_dropout: Option<f64>,
+ pub model_type: Option<String>,
+}
+
+impl Default for Config {
+ fn default() -> Self {
Self {
+ vocab_size: 30522,
+ hidden_size: 768,
+ num_hidden_layers: 12,
+ num_attention_heads: 12,
+ intermediate_size: 3072,
+ hidden_act: HiddenAct::Gelu,
+ hidden_dropout_prob: 0.1,
+ max_position_embeddings: 512,
+ type_vocab_size: 2,
+ initializer_range: 0.02,
+ layer_norm_eps: 1e-12,
+ pad_token_id: 0,
+ position_embedding_type: PositionEmbeddingType::Absolute,
+ use_cache: true,
+ classifier_dropout: None,
+ model_type: Some("bert".to_string()),
+ }
+ }
+}
+
+impl Config {
+ fn _all_mini_lm_l6_v2() -> Self {
+ // https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/blob/main/config.json
+ Self {
+ vocab_size: 30522,
+ hidden_size: 384,
+ num_hidden_layers: 6,
+ num_attention_heads: 12,
+ intermediate_size: 1536,
+ hidden_act: HiddenAct::Gelu,
+ hidden_dropout_prob: 0.1,
+ max_position_embeddings: 512,
+ type_vocab_size: 2,
+ initializer_range: 0.02,
+ layer_norm_eps: 1e-12,
+ pad_token_id: 0,
+ position_embedding_type: PositionEmbeddingType::Absolute,
+ use_cache: true,
+ classifier_dropout: None,
+ model_type: Some("bert".to_string()),
+ }
+ }
+}
+
+struct Dropout {
+ #[allow(dead_code)]
+ pr: f64,
+}
+
+impl Dropout {
+ fn new(pr: f64) -> Self {
+ Self { pr }
+ }
+}
+
+impl Module for Dropout {
+ fn forward(&self, x: &Tensor) -> Result<Tensor> {
+ // TODO
+ Ok(x.clone())
+ }
+}
+
+// https://github.com/huggingface/transformers/blob/6eedfa6dd15dc1e22a55ae036f681914e5a0d9a1/src/transformers/models/bert/modeling_bert.py#L180
+struct BertEmbeddings {
+ word_embeddings: Embedding,
+ position_embeddings: Option<Embedding>,
+ token_type_embeddings: Embedding,
+ layer_norm: LayerNorm,
+ dropout: Dropout,
+ span: tracing::Span,
+}
+
+impl BertEmbeddings {
+ fn load(vb: VarBuilder, config: &Config) -> Result<Self> {
+ let word_embeddings = embedding(
+ config.vocab_size,
+ config.hidden_size,
+ vb.pp("word_embeddings"),
+ )?;
+ let position_embeddings = embedding(
+ config.max_position_embeddings,
+ config.hidden_size,
+ vb.pp("position_embeddings"),
+ )?;
+ let token_type_embeddings = embedding(
+ config.type_vocab_size,
+ config.hidden_size,
+ vb.pp("token_type_embeddings"),
+ )?;
+ let layer_norm = layer_norm(
+ config.hidden_size,
+ config.layer_norm_eps,
+ vb.pp("LayerNorm"),
+ )?;
+ Ok(Self {
word_embeddings,
- position_embeddings,
+ position_embeddings: Some(position_embeddings),
token_type_embeddings,
layer_norm,
+ dropout: Dropout::new(config.hidden_dropout_prob),
+ span: tracing::span!(tracing::Level::TRACE, "embeddings"),
+ })
+ }
+
+ fn forward(&self, input_ids: &Tensor, token_type_ids: &Tensor) -> Result<Tensor> {
+ let _enter = self.span.enter();
+ let (_bsize, seq_len) = input_ids.dims2()?;
+ let input_embeddings = self.word_embeddings.forward(input_ids)?;
+ let token_type_embeddings = self.token_type_embeddings.forward(token_type_ids)?;
+ let mut embeddings = (&input_embeddings + token_type_embeddings)?;
+ if let Some(position_embeddings) = &self.position_embeddings {
+ // TODO: Proper absolute positions?
+ let position_ids = (0..seq_len as u32).collect::<Vec<_>>();
+ let position_ids = Tensor::new(&position_ids[..], input_ids.device())?;
+ embeddings = embeddings.broadcast_add(&position_embeddings.forward(&position_ids)?)?
}
+ let embeddings = self.layer_norm.forward(&embeddings)?;
+ let embeddings = self.dropout.forward(&embeddings)?;
+ Ok(embeddings)
}
+}
- pub fn forward(
- &mut self,
- input_ids: &Tensor<1>,
- token_type_ids: &Tensor<1>,
- position_ids: &Tensor<1>,
- ) -> Tensor<2> {
- let word_embeddings = self.word_embeddings.forward(input_ids);
- let position_embeddings = self.position_embeddings.forward(position_ids);
- let token_type_embeddings = self.token_type_embeddings.forward(token_type_ids);
+struct BertSelfAttention {
+ query: Linear,
+ key: Linear,
+ value: Linear,
+ dropout: Dropout,
+ num_attention_heads: usize,
+ attention_head_size: usize,
+ span: tracing::Span,
+ span_softmax: tracing::Span,
+}
+
+impl BertSelfAttention {
+ fn load(vb: VarBuilder, config: &Config) -> Result<Self> {
+ let attention_head_size = config.hidden_size / config.num_attention_heads;
+ let all_head_size = config.num_attention_heads * attention_head_size;
+ let dropout = Dropout::new(config.hidden_dropout_prob);
+ let hidden_size = config.hidden_size;
+ let query = linear(hidden_size, all_head_size, vb.pp("query"))?;
+ let value = linear(hidden_size, all_head_size, vb.pp("value"))?;
+ let key = linear(hidden_size, all_head_size, vb.pp("key"))?;
+ Ok(Self {
+ query,
+ key,
+ value,
+ dropout,
+ num_attention_heads: config.num_attention_heads,
+ attention_head_size,
+ span: tracing::span!(tracing::Level::TRACE, "self-attn"),
+ span_softmax: tracing::span!(tracing::Level::TRACE, "softmax"),
+ })
+ }
+
+ fn transpose_for_scores(&self, xs: &Tensor) -> Result<Tensor> {
+ let mut new_x_shape = xs.dims().to_vec();
+ new_x_shape.pop();
+ new_x_shape.push(self.num_attention_heads);
+ new_x_shape.push(self.attention_head_size);
+ let xs = xs.reshape(new_x_shape.as_slice())?.transpose(1, 2)?;
+ xs.contiguous()
+ }
+}
+
+impl BertSelfAttention {
+ pub fn forward(&self, hidden_states: &Tensor, attention_mask: &Tensor) -> Result<Tensor> {
+ let _enter = self.span.enter();
+ let query_layer = self.query.forward(hidden_states)?;
+ let key_layer = self.key.forward(hidden_states)?;
+ let value_layer = self.value.forward(hidden_states)?;
+
+ let query_layer = self.transpose_for_scores(&query_layer)?;
+ let key_layer = self.transpose_for_scores(&key_layer)?;
+ let value_layer = self.transpose_for_scores(&value_layer)?;
+
+ let attention_scores = query_layer.matmul(&key_layer.t()?)?;
+ let attention_scores = (attention_scores / (self.attention_head_size as f64).sqrt())?;
+ let attention_scores = attention_scores.broadcast_add(&attention_mask)?;
+
+ let attention_probs = {
+ let _enter_sm = self.span_softmax.enter();
+ candle_nn::ops::softmax(&attention_scores, candle_core::D::Minus1)?
+ };
+ let attention_probs = self.dropout.forward(&attention_probs)?;
+
+ let context_layer = attention_probs.matmul(&value_layer)?;
+ let context_layer = context_layer.transpose(1, 2)?.contiguous()?;
+ let context_layer = context_layer.flatten_from(candle_core::D::Minus2)?;
+ Ok(context_layer)
+ }
+}
+
+struct BertSelfOutput {
+ dense: Linear,
+ layer_norm: LayerNorm,
+ dropout: Dropout,
+ span: tracing::Span,
+}
+
+impl BertSelfOutput {
+ fn load(vb: VarBuilder, config: &Config) -> Result<Self> {
+ let dense = linear(config.hidden_size, config.hidden_size, vb.pp("dense"))?;
+ let layer_norm = layer_norm(
+ config.hidden_size,
+ config.layer_norm_eps,
+ vb.pp("LayerNorm"),
+ )?;
+ let dropout = Dropout::new(config.hidden_dropout_prob);
+ Ok(Self {
+ dense,
+ layer_norm,
+ dropout,
+ span: tracing::span!(tracing::Level::TRACE, "self-out"),
+ })
+ }
+
+ fn forward(&self, hidden_states: &Tensor, input_tensor: &Tensor) -> Result<Tensor> {
+ let _enter = self.span.enter();
+ let hidden_states = self.dense.forward(hidden_states)?;
+ let hidden_states = self.dropout.forward(&hidden_states)?;
+ self.layer_norm.forward(&(hidden_states + input_tensor)?)
+ }
+}
+
+// https://github.com/huggingface/transformers/blob/6eedfa6dd15dc1e22a55ae036f681914e5a0d9a1/src/transformers/models/bert/modeling_bert.py#L392
+struct BertAttention {
+ self_attention: BertSelfAttention,
+ self_output: BertSelfOutput,
+ span: tracing::Span,
+}
- let embeddings = word_embeddings + position_embeddings + token_type_embeddings;
+impl BertAttention {
+ fn load(vb: VarBuilder, config: &Config) -> Result<Self> {
+ let self_attention = BertSelfAttention::load(vb.pp("self"), config)?;
+ let self_output = BertSelfOutput::load(vb.pp("output"), config)?;
+ Ok(Self {
+ self_attention,
+ self_output,
+ span: tracing::span!(tracing::Level::TRACE, "attn"),
+ })
+ }
+}
+
+impl BertAttention {
+ fn forward(&self, hidden_states: &Tensor, attention_mask: &Tensor) -> Result<Tensor> {
+ let _enter = self.span.enter();
+ let self_outputs = self.self_attention.forward(hidden_states, attention_mask)?;
+ let attention_output = self.self_output.forward(&self_outputs, hidden_states)?;
+ Ok(attention_output)
+ }
+}
+
+// https://github.com/huggingface/transformers/blob/6eedfa6dd15dc1e22a55ae036f681914e5a0d9a1/src/transformers/models/bert/modeling_bert.py#L441
+struct BertIntermediate {
+ dense: Linear,
+ intermediate_act: HiddenActLayer,
+ span: tracing::Span,
+}
+
+impl BertIntermediate {
+ fn load(vb: VarBuilder, config: &Config) -> Result<Self> {
+ let dense = linear(config.hidden_size, config.intermediate_size, vb.pp("dense"))?;
+ Ok(Self {
+ dense,
+ intermediate_act: HiddenActLayer::new(config.hidden_act),
+ span: tracing::span!(tracing::Level::TRACE, "inter"),
+ })
+ }
+}
+
+impl Module for BertIntermediate {
+ fn forward(&self, hidden_states: &Tensor) -> Result<Tensor> {
+ let _enter = self.span.enter();
+ let hidden_states = self.dense.forward(hidden_states)?;
+ let ys = self.intermediate_act.forward(&hidden_states)?;
+ Ok(ys)
+ }
+}
+
+// https://github.com/huggingface/transformers/blob/6eedfa6dd15dc1e22a55ae036f681914e5a0d9a1/src/transformers/models/bert/modeling_bert.py#L456
+struct BertOutput {
+ dense: Linear,
+ layer_norm: LayerNorm,
+ dropout: Dropout,
+ span: tracing::Span,
+}
+
+impl BertOutput {
+ fn load(vb: VarBuilder, config: &Config) -> Result<Self> {
+ let dense = linear(config.intermediate_size, config.hidden_size, vb.pp("dense"))?;
+ let layer_norm = layer_norm(
+ config.hidden_size,
+ config.layer_norm_eps,
+ vb.pp("LayerNorm"),
+ )?;
+ let dropout = Dropout::new(config.hidden_dropout_prob);
+ Ok(Self {
+ dense,
+ layer_norm,
+ dropout,
+ span: tracing::span!(tracing::Level::TRACE, "out"),
+ })
+ }
+
+ fn forward(&self, hidden_states: &Tensor, input_tensor: &Tensor) -> Result<Tensor> {
+ let _enter = self.span.enter();
+ let hidden_states = self.dense.forward(hidden_states)?;
+ let hidden_states = self.dropout.forward(&hidden_states)?;
+ self.layer_norm.forward(&(hidden_states + input_tensor)?)
+ }
+}
+
+// https://github.com/huggingface/transformers/blob/6eedfa6dd15dc1e22a55ae036f681914e5a0d9a1/src/transformers/models/bert/modeling_bert.py#L470
+struct BertLayer {
+ attention: BertAttention,
+ intermediate: BertIntermediate,
+ output: BertOutput,
+ span: tracing::Span,
+}
+
+impl BertLayer {
+ fn load(vb: VarBuilder, config: &Config) -> Result<Self> {
+ let attention = BertAttention::load(vb.pp("attention"), config)?;
+ let intermediate = BertIntermediate::load(vb.pp("intermediate"), config)?;
+ let output = BertOutput::load(vb.pp("output"), config)?;
+ Ok(Self {
+ attention,
+ intermediate,
+ output,
+ span: tracing::span!(tracing::Level::TRACE, "layer"),
+ })
+ }
+}
+
+impl BertLayer {
+ fn forward(&self, hidden_states: &Tensor, attention_mask: &Tensor) -> Result<Tensor> {
+ let _enter = self.span.enter();
+ let attention_output = self.attention.forward(hidden_states, attention_mask)?;
+ // TODO: Support cross-attention?
+ // https://github.com/huggingface/transformers/blob/6eedfa6dd15dc1e22a55ae036f681914e5a0d9a1/src/transformers/models/bert/modeling_bert.py#L523
+ // TODO: Support something similar to `apply_chunking_to_forward`?
+ let intermediate_output = self.intermediate.forward(&attention_output)?;
+ let layer_output = self
+ .output
+ .forward(&intermediate_output, &attention_output)?;
+ Ok(layer_output)
+ }
+}
+
+// https://github.com/huggingface/transformers/blob/6eedfa6dd15dc1e22a55ae036f681914e5a0d9a1/src/transformers/models/bert/modeling_bert.py#L556
+struct BertEncoder {
+ layers: Vec<BertLayer>,
+ span: tracing::Span,
+}
- self.layer_norm.forward(&embeddings)
+impl BertEncoder {
+ fn load(vb: VarBuilder, config: &Config) -> Result<Self> {
+ let layers = (0..config.num_hidden_layers)
+ .map(|index| BertLayer::load(vb.pp(&format!("layer.{index}")), config))
+ .collect::<Result<Vec<_>>>()?;
+ let span = tracing::span!(tracing::Level::TRACE, "encoder");
+ Ok(BertEncoder { layers, span })
}
}
-#[cfg(test)]
-mod tests {
- use super::*;
+impl BertEncoder {
+ fn forward(&self, hidden_states: &Tensor, attention_mask: &Tensor) -> Result<Tensor> {
+ let _enter = self.span.enter();
+ let mut hidden_states = hidden_states.clone();
+ // Use a loop rather than a fold as it's easier to modify when adding debug/...
+ for layer in self.layers.iter() {
+ hidden_states = layer.forward(&hidden_states, attention_mask)?
+ }
+ Ok(hidden_states)
+ }
+}
+
+// https://github.com/huggingface/transformers/blob/6eedfa6dd15dc1e22a55ae036f681914e5a0d9a1/src/transformers/models/bert/modeling_bert.py#L654
+pub struct BertPooler {
+ dense: Linear,
+}
+
+impl BertPooler {
+ fn load(vb: VarBuilder, config: &Config) -> Result<Self> {
+ let dense = linear(config.hidden_size, config.hidden_size, vb.pp("dense"))?;
+ Ok(Self { dense })
+ }
+}
+
+impl Module for BertPooler {
+ fn forward(&self, hidden_states: &Tensor) -> Result<Tensor> {
+ let first_token_tensor = hidden_states.get_on_dim(1, 0)?;
+ let pooled_output = self.dense.forward(&first_token_tensor)?;
+ pooled_output.tanh()
+ }
+}
+
+// https://github.com/huggingface/transformers/blob/6eedfa6dd15dc1e22a55ae036f681914e5a0d9a1/src/transformers/models/bert/modeling_bert.py#L874
+pub struct BertModel {
+ embeddings: BertEmbeddings,
+ encoder: BertEncoder,
+ pooler: Option<BertPooler>,
+ pub device: Device,
+ span: tracing::Span,
+}
+
+impl BertModel {
+ pub fn load(vb: VarBuilder, config: &Config) -> Result<Self> {
+ let (embeddings, encoder) = match (
+ BertEmbeddings::load(vb.pp("embeddings"), config),
+ BertEncoder::load(vb.pp("encoder"), config),
+ ) {
+ (Ok(embeddings), Ok(encoder)) => (embeddings, encoder),
+ (Err(err), _) | (_, Err(err)) => {
+ if let Some(model_type) = &config.model_type {
+ if let (Ok(embeddings), Ok(encoder)) = (
+ BertEmbeddings::load(vb.pp(&format!("{model_type}.embeddings")), config),
+ BertEncoder::load(vb.pp(&format!("{model_type}.encoder")), config),
+ ) {
+ (embeddings, encoder)
+ } else {
+ return Err(err);
+ }
+ } else {
+ return Err(err);
+ }
+ }
+ };
+
+ let pooler = BertPooler::load(vb.pp("pooler"), config)
+ .or_else(|e| {
+ if let Some(model_type) = &config.model_type {
+ BertPooler::load(vb.pp(&format!("{model_type}.pooler")), config)
+ } else {
+ Err(e)
+ }
+ })
+ .ok();
+
+ Ok(Self {
+ embeddings,
+ encoder,
+ pooler,
+ device: vb.device().clone(),
+ span: tracing::span!(tracing::Level::TRACE, "model"),
+ })
+ }
+
+ pub fn set_pooler(&mut self, pooler: Option<BertPooler>) {
+ self.pooler = pooler;
+ }
+
+ fn extended_attention_mask(
+ &self,
+ attention_mask: &Tensor,
+ input_ids_dims: &[usize],
+ ) -> Result<Tensor> {
+ let bs = input_ids_dims[0];
+ let seq = input_ids_dims[input_ids_dims.len() - 1];
+ let attention_mask = attention_mask.reshape(&[bs, 1, 1, seq])?;
+
+ // TODO: subtract min value of dtype instead of 1e4
+ // i.e. something like: (1.0 - attention_mask)? * dtype.min_value()?
+ (1.0 - attention_mask)? * -1e4
+ }
+
+ pub fn forward(
+ &self,
+ input_ids: &Tensor,
+ token_type_ids: &Tensor,
+ attention_mask: &Tensor,
+ ) -> Result<Tensor> {
+ let _enter = self.span.enter();
+ let dims = input_ids.dims();
+
+ if dims.len() <= 1 {
+ return Err(candle_core::Error::UnexpectedNumberOfDims {
+ expected: 2,
+ got: dims.len(),
+ shape: input_ids.shape().clone(),
+ });
+ }
+
+ let attention_mask = self.extended_attention_mask(attention_mask, &dims)?;
+
+ let embedding_output = self.embeddings.forward(input_ids, token_type_ids)?;
+ let sequence_output = self.encoder.forward(&embedding_output, &attention_mask)?;
+
+ match &self.pooler {
+ Some(pooler) => pooler.forward(&sequence_output),
+ None => Ok(sequence_output),
+ }
+ }
}
diff --git a/crates/core/src/models/mod.rs b/crates/core/src/models/mod.rs
index f94d32f4..01b31e25 100644
--- a/crates/core/src/models/mod.rs
+++ b/crates/core/src/models/mod.rs
@@ -1 +1 @@
-mod bert;
+pub mod bert;
diff --git a/crates/core/src/ranking/models/cross_encoder.rs b/crates/core/src/ranking/models/cross_encoder.rs
index a434140e..2d6c8e51 100644
--- a/crates/core/src/ranking/models/cross_encoder.rs
+++ b/crates/core/src/ranking/models/cross_encoder.rs
@@ -16,127 +16,134 @@
use anyhow::anyhow;
use anyhow::Result;
+use candle_core::Module;
+use candle_core::{Device, Tensor};
+use candle_nn::Linear;
+use candle_nn::VarBuilder;
use std::path::Path;
+use tokenizers::PaddingParams;
+use tokenizers::TruncationParams;
-#[cfg(feature = "libtorch")]
-pub use model::*;
+use crate::models::bert;
+use crate::models::bert::BertModel;
-#[cfg(feature = "libtorch")]
-mod model {
- use super::*;
-
- use itertools::Itertools;
- use tch::Tensor;
- use tokenizers::PaddingParams;
- use tokenizers::TruncationParams;
+const TRUNCATE_INPUT: usize = 128;
- const TRUNCATE_INPUT: usize = 128;
+pub struct CrossEncoderModel {
+ tokenizer: tokenizers::Tokenizer,
+ encoder: BertModel,
+ classifier: Linear,
+ device: Device,
+ dtype: candle_core::DType,
+}
- pub struct CrossEncoderModel {
- tokenizer: tokenizers::Tokenizer,
- model: tch::CModule,
+impl CrossEncoderModel {
+ pub fn open<P: AsRef<Path>>(folder: P) -> Result<Self> {
+ let device = Device::Cpu;
+ let dtype = candle_core::DType::F16;
+
+ let truncation = TruncationParams {
+ max_length: TRUNCATE_INPUT,
+ ..Default::default()
+ };
+
+ let padding = PaddingParams {
+ ..Default::default()
+ };
+
+ let mut tokenizer =
+ tokenizers::Tokenizer::from_file(folder.as_ref().join("tokenizer.json"))
+ .map_err(|_| anyhow!("couldn't open tokenizer"))?;
+
+ tokenizer
+ .with_truncation(Some(truncation))
+ .map_err(|_| anyhow!("tokenizer truncation settings"))?;
+ tokenizer.with_padding(Some(padding));
+
+ let vb = unsafe {
+ VarBuilder::from_mmaped_safetensors(
+ &[folder.as_ref().join("model.safetensors")],
+ dtype,
+ &device,
+ )?
+ };
+ let config = std::fs::read_to_string(folder.as_ref().join("config.json"))?;
+ let mut config: bert::Config = serde_json::from_str(&config)?;
+ config.hidden_act = bert::HiddenAct::GeluApproximate;
+
+ let classifier: Linear = candle_nn::linear(config.hidden_size, 1, vb.pp("classifier"))?;
+
+ // all tensors can be loaded with (useful for debugging):
+ // candle_core::safetensors::load(folder.as_ref().join("model.safetensors"), &device)
+
+ let encoder = BertModel::load(vb, &config)?;
+
+ Ok(Self {
+ tokenizer,
+ encoder,
+ classifier,
+ device,
+ dtype,
+ })
}
+}
- impl CrossEncoderModel {
- pub fn open<P: AsRef<Path>>(folder: P) -> Result<Self> {
- let truncation = TruncationParams {
- max_length: TRUNCATE_INPUT,
- ..Default::default()
- };
-
- let padding = PaddingParams {
- ..Default::default()
- };
-
- let mut tokenizer =
- tokenizers::Tokenizer::from_file(folder.as_ref().join("tokenizer.json"))
- .map_err(|_| anyhow!("couldn't open tokenizer"))?;
-
- tokenizer
- .with_truncation(Some(truncation))
- .map_err(|_| anyhow!("tokenizer truncation settings"))?;
- tokenizer.with_padding(Some(padding));
-
- let model = tch::CModule::load(folder.as_ref().join("model.pt"))?;
-
- Ok(Self { tokenizer, model })
+impl CrossEncoder for CrossEncoderModel {
+ fn run(&self, query: &str, bodies: &[String]) -> Vec<f64> {
+ if bodies.is_empty() {
+ return Vec::new();
}
- }
- impl CrossEncoder for CrossEncoderModel {
- fn run(&self, query: &str, bodies: &[String]) -> Vec<f64> {
- if bodies.is_empty() {
- return Vec::new();
- }
-
- let bs = bodies.len();
-
- let input: Vec<_> = bodies
- .iter()
- .map(|body| {
- (
- query.to_string(),
- body.split_whitespace()
- .take(TRUNCATE_INPUT)
- .collect::<String>(),
- )
- })
- .collect();
-
- let encoded = self.tokenizer.encode_batch(input, true).unwrap();
-
- let num_tokens = encoded
- .iter()
- .map(|enc| enc.get_ids().len())
- .max()
- .unwrap_or(0);
-
- let ids = encoded
- .iter()
- .flat_map(|enc| enc.get_ids().iter().map(|i| *i as i64).take(TRUNCATE_INPUT))
- .collect_vec();
-
- let attention_mask = encoded
- .iter()
- .flat_map(|enc| {
- enc.get_attention_mask()
- .iter()
- .map(|i| *i as i64)
- .take(TRUNCATE_INPUT)
- })
- .collect_vec();
-
- let type_ids = encoded
- .iter()
- .flat_map(|enc| {
- enc.get_type_ids()
- .iter()
- .map(|i| *i as i64)
+ let input: Vec<_> = bodies
+ .iter()
+ .map(|body| {
+ (
+ query.to_string(),
+ body.split_whitespace()
.take(TRUNCATE_INPUT)
- })
- .collect_vec();
-
- let ids = Tensor::from_slice(&ids).reshape([bs as i64, num_tokens as i64]);
- let attention_mask =
- Tensor::from_slice(&attention_mask).reshape([bs as i64, num_tokens as i64]);
- let type_ids = Tensor::from_slice(&type_ids).reshape([bs as i64, num_tokens as i64]);
-
- let output = self
- .model
- .forward_ts(&[ids, attention_mask, type_ids])
- .unwrap();
-
- let mut res = Vec::with_capacity(bs);
-
- for i in 0..bs {
- let s = output.double_value(&[i as i64, 0]);
- let s = s.exp();
- let s = s / (s + 1.0);
- res.push(s);
- }
-
- res
- }
+ .collect::<String>(),
+ )
+ })
+ .collect();
+
+ let encoded = self.tokenizer.encode_batch(input, true).unwrap();
+
+ let ids = encoded
+ .iter()
+ .map(|enc| Tensor::new(enc.get_ids(), &self.device).map_err(|e| anyhow!(e)))
+ .collect::<Result<Vec<_>>>()
+ .unwrap();
+ let input_ids = Tensor::stack(&ids, 0).unwrap();
+
+ let token_type_ids = input_ids.zeros_like().unwrap();
+
+ let attention_mask = encoded
+ .iter()
+ .map(|enc| Tensor::new(enc.get_attention_mask(), &self.device).map_err(|e| anyhow!(e)))
+ .collect::<Result<Vec<_>>>()
+ .unwrap();
+ let attention_mask = Tensor::stack(&attention_mask, 0)
+ .unwrap()
+ .to_dtype(self.dtype)
+ .unwrap();
+
+ let logits = self
+ .encoder
+ .forward(&input_ids, &token_type_ids, &attention_mask)
+ .unwrap();
+
+ let scores = self
+ .classifier
+ .forward(&logits)
+ .unwrap()
+ .squeeze(1)
+ .unwrap()
+ .to_dtype(candle_core::DType::F64)
+ .unwrap();
+
+ let scores = candle_nn::ops::sigmoid(&scores).unwrap();
+
+ scores.to_vec1().unwrap()
}
}
@@ -153,12 +160,15 @@ impl CrossEncoder for DummyCrossEncoder {
}
#[cfg(test)]
-#[cfg(feature = "libtorch")]
mod tests {
use super::*;
#[test]
fn sanity_check() {
+ if !Path::new("../../data/cross_encoder").exists() {
+ return;
+ }
+
let model = CrossEncoderModel::open("../../data/cross_encoder")
.expect("Failed to find cross-encoder model");
@@ -177,14 +187,14 @@ mod tests {
);
}
- assert!(
- model.run(
- "how many people live in paris",
- &["there are currently 1234 people living in paris".to_string()]
- )[0] > model.run(
- "how many people live in paris",
- &["I really like cake and my favorite cake is probably brownie".to_string()]
- )[0]
+ let res = model.run(
+ "how many people live in paris",
+ &[
+ "there are currently 1234 people living in paris".to_string(),
+ "I really like cake and my favorite cake is probably brownie".to_string(),
+ ],
);
+
+ assert!(res[0] > res[1]);
}
}
diff --git a/crates/core/src/ranking/models/mod.rs b/crates/core/src/ranking/models/mod.rs
index a2b52c47..41e9906e 100644
--- a/crates/core/src/ranking/models/mod.rs
+++ b/crates/core/src/ranking/models/mod.rs
@@ -1,19 +1,3 @@
-// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
pub mod cross_encoder;
pub mod lambdamart;
pub mod linear;
diff --git a/crates/core/src/summarizer.rs b/crates/core/src/summarizer.rs
index 82d28299..901a8c0c 100644
--- a/crates/core/src/summarizer.rs
+++ b/crates/core/src/summarizer.rs
@@ -14,6 +14,9 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use anyhow::anyhow;
+use candle_core::{Device, Tensor};
+use candle_nn::VarBuilder;
use futures::stream::Stream;
use std::{
cmp::Reverse,
@@ -23,9 +26,12 @@ use std::{
};
use tokio_stream::StreamExt;
-use crate::{llm_utils::OpenAiApi, Result};
+use crate::{
+ llm_utils::OpenAiApi,
+ models::bert::{self, BertModel},
+ Result,
+};
use itertools::{intersperse, Itertools};
-use tch::Tensor;
use tokenizers::{PaddingParams, TruncationParams};
use crate::ceil_char_boundary;
@@ -284,12 +290,17 @@ impl Summarizer {
}
pub struct DualEncoder {
- model: tch::CModule,
+ model: BertModel,
tokenizer: tokenizers::Tokenizer,
+ device: Device,
+ dtype: candle_core::DType,
}
impl DualEncoder {
pub fn open<P: AsRef<Path>>(folder: P) -> Result<Self> {
+ let device = Device::Cpu;
+ let dtype = candle_core::DType::F16;
+
let truncation = TruncationParams {
max_length: 256,
..Default::default()
@@ -305,36 +316,62 @@ impl DualEncoder {
tokenizer.with_truncation(Some(truncation)).unwrap();
tokenizer.with_padding(Some(padding));
- let model = tch::CModule::load(folder.as_ref().join("model.pt"))?;
- Ok(Self { model, tokenizer })
+ let vb = unsafe {
+ VarBuilder::from_mmaped_safetensors(
+ &[folder.as_ref().join("model.safetensors")],
+ dtype,
+ &device,
+ )?
+ };
+ let config = std::fs::read_to_string(folder.as_ref().join("config.json"))?;
+ let mut config: bert::Config = serde_json::from_str(&config)?;
+ config.hidden_act = bert::HiddenAct::GeluApproximate;
+
+ // all tensors can be loaded with (useful for debugging):
+ // candle_core::safetensors::load(folder.as_ref().join("model.safetensors"), &device)
+
+ let mut model = BertModel::load(vb, &config)?;
+ model.set_pooler(None); // model should use mean pooling
+
+ Ok(Self {
+ model,
+ tokenizer,
+ device,
+ dtype,
+ })
}
- fn embed(&self, text: &str) -> Result<Tensor> {
- let query = self.tokenizer.encode(text, false).unwrap();
+ pub fn embed(&self, texts: &[&str]) -> Result<Tensor> {
+ let enc = self
+ .tokenizer
+ .encode_batch(texts.to_vec(), true)
+ .map_err(|e| anyhow!(e))?;
- let ids = query
- .get_ids()
+ let ids = enc
.iter()
- .map(|&id| id as i64)
- .collect::<Vec<_>>();
+ .map(|enc| Tensor::new(enc.get_ids(), &self.device).map_err(|e| anyhow!(e)))
+ .collect::<Result<Vec<_>>>()?;
- let types = query
- .get_type_ids()
- .iter()
- .map(|&id| id as i64)
- .collect::<Vec<_>>();
+ let input_ids = Tensor::stack(&ids, 0)?;
+
+ let token_type_ids = input_ids.zeros_like()?;
- let mask = query
- .get_attention_mask()
+ let attention_mask = enc
.iter()
- .map(|&id| id as i64)
- .collect::<Vec<_>>();
+ .map(|enc| Tensor::new(enc.get_attention_mask(), &self.device).map_err(|e| anyhow!(e)))
+ .collect::<Result<Vec<_>>>()?;
+ let attention_mask = Tensor::stack(&attention_mask, 0)?.to_dtype(self.dtype)?;
- let ids = Tensor::from_slice(&ids).reshape([1, -1]);
- let types = Tensor::from_slice(&types).reshape([1, -1]);
- let mask = Tensor::from_slice(&mask).reshape([1, -1]);
+ let emb = self
+ .model
+ .forward(&input_ids, &token_type_ids, &attention_mask)?;
- Ok(self.model.forward_ts(&[ids, types, mask])?.squeeze())
+ let (_n_sentence, n_tokens, _hidden_size) = emb.dims3()?;
+
+ let emb = (emb.sum(1)? / (n_tokens as f64))?; // mean pooling
+ let emb = emb.broadcast_div(&emb.sqr()?.sum_keepdim(1)?.sqrt()?)?; // l2 normalization
+
+ Ok(emb)
}
}
@@ -344,17 +381,25 @@ impl PassageScorer for DualEncoder {
type PassageEmbedding = Tensor;
fn embed_query(&self, query: &str) -> Option<Self::QueryEmbedding> {
- self.embed(query).ok()
+ self.embed(&[query]).ok()
}
fn embed_passage(&self, passage: &str) -> Option<Self::PassageEmbedding> {
- self.embed(passage).ok()
+ self.embed(&[passage]).ok()
}
fn score(&self, query: &Self::QueryEmbedding, passage: &Self::PassageEmbedding) -> f32 {
- let score = query.dot(&passage.transpose(0, 0));
-
- score.double_value(&[]) as f32
+ query
+ .matmul(&passage.t().unwrap())
+ .unwrap()
+ .get(0)
+ .unwrap()
+ .squeeze(0)
+ .unwrap()
+ .to_dtype(candle_core::DType::F32)
+ .unwrap()
+ .to_vec0()
+ .unwrap()
}
}
@@ -497,4 +542,24 @@ mod tests {
assert_eq!(p, &text[range]);
}
}
+
+ #[test]
+ fn test_dual_encoder() {
+ if !Path::new("../../data/summarizer/dual_encoder").exists() {
+ return;
+ }
+
+ let model =
+ DualEncoder::open("../../data/summarizer/dual_encoder").expect("Failed to load model");
+ let query = "What is the capital of France?";
+ let pos = "The capital of France is Paris.";
+ let neg = "The best baguette in Paris can be found at Boulangerie Pichard.";
+
+ let query_emb = model.embed_query(query).unwrap();
+ let pos_emb = model.embed_passage(pos).unwrap();
+ let neg_emb = model.embed_passage(neg).unwrap();
+
+ assert!(model.score(&query_emb, &pos_emb) > 0.0);
+ assert!(model.score(&query_emb, &pos_emb) > model.score(&query_emb, &neg_emb));
+ }
}
diff --git a/crates/ggml-sys/Cargo.toml b/crates/ggml-sys/Cargo.toml
deleted file mode 100644
index fc95fe70..00000000
--- a/crates/ggml-sys/Cargo.toml
+++ /dev/null
@@ -1,11 +0,0 @@
-[package]
-name = "ggml-sys"
-authors = ['Stract <[email protected]>']
-version = "0.1.0"
-edition = "2021"
-license = "MIT"
-
-[build-dependencies]
-bindgen = { workspace = true }
-cmake = { workspace = true }
-cc = { workspace = true }
diff --git a/crates/ggml-sys/build.rs b/crates/ggml-sys/build.rs
deleted file mode 100644
index 86d6928a..00000000
--- a/crates/ggml-sys/build.rs
+++ /dev/null
@@ -1,45 +0,0 @@
-use std::env;
-use std::path::PathBuf;
-
-fn generate_bindings() {
- let out_path = PathBuf::from(env::var("OUT_DIR").unwrap());
- bindgen::builder()
- .header("ggml-src/include/ggml/ggml.h")
- .header("ggml-src/include/ggml/ggml-alloc.h")
- .header("ggml-src/include/ggml/ggml-backend.h")
- .parse_callbacks(Box::new(bindgen::CargoCallbacks::new()))
- .generate()
- .expect("Unable to generate bindings")
- .write_to_file(out_path.join("bindings.rs"))
- .expect("Couldn't write bindings!");
-}
-
-fn main() {
- let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap();
-
- generate_bindings();
-
- let ggml_src = PathBuf::from("ggml-src")
- .join("src")
- .canonicalize()
- .unwrap();
- let mut build = cmake::Config::new(ggml_src);
-
- build.define("GGML_STATIC", "ON");
-
- if target_os == "linux" {
- build.define("GGML_OPENBLAS", "ON");
- println!("cargo:rustc-link-lib=openblas");
- }
-
- if target_os == "macos" {
- println!("cargo:rustc-link-lib=framework=Accelerate");
- }
-
- let dst = build.build();
-
- println!("cargo:rustc-link-search={}/lib", dst.display());
-
- println!("cargo:rerun-if-changed=ggml-src");
- println!("cargo:rustc-link-lib=static=ggml");
-}
diff --git a/crates/ggml-sys/ggml-src b/crates/ggml-sys/ggml-src
deleted file mode 160000
index ca7e136d..00000000
--- a/crates/ggml-sys/ggml-src
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit ca7e136d7c4cd56399ea399b0757fb3806d91e80
diff --git a/crates/ggml-sys/src/lib.rs b/crates/ggml-sys/src/lib.rs
deleted file mode 100644
index 514d755c..00000000
--- a/crates/ggml-sys/src/lib.rs
+++ /dev/null
@@ -1,35 +0,0 @@
-#![allow(non_upper_case_globals)]
-#![allow(non_camel_case_types)]
-#![allow(non_snake_case)]
-
-include!(concat!(env!("OUT_DIR"), "/bindings.rs"));
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn sanity() {
- unsafe {
- let params = ggml_init_params {
- mem_size: 128 * 1024 * 1024,
- mem_buffer: std::ptr::null_mut(),
- no_alloc: false,
- };
-
- let ctx = ggml_init(params);
- let a = ggml_new_f32(ctx, 2.0);
- let b = ggml_new_f32(ctx, 2.0);
-
- let c = ggml_add(ctx, a, b);
-
- let gf = ggml_new_graph(ctx);
- ggml_build_forward_expand(gf, c);
-
- ggml_graph_compute_with_ctx(ctx, gf, 1);
-
- let res = ggml_get_f32_1d(c, 0);
- assert_eq!(res, 4.0);
- }
- }
-}
diff --git a/crates/ggml/Cargo.toml b/crates/ggml/Cargo.toml
deleted file mode 100644
index 329ba104..00000000
--- a/crates/ggml/Cargo.toml
+++ /dev/null
@@ -1,11 +0,0 @@
-[package]
-name = "ggml"
-authors = ['Stract <[email protected]>']
-version = "0.1.0"
-edition = "2021"
-license = "MIT"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-ggml-sys = {path = "../ggml-sys"}
diff --git a/crates/ggml/src/context.rs b/crates/ggml/src/context.rs
deleted file mode 100644
index a1f83ab2..00000000
--- a/crates/ggml/src/context.rs
+++ /dev/null
@@ -1,125 +0,0 @@
-use std::{ptr::NonNull, sync::Arc};
-
-use crate::{Dims, Tensor, ValidDims};
-
-pub(crate) struct InnerContext {
- ctx: NonNull<ggml_sys::ggml_context>,
-}
-
-impl InnerContext {
- fn new(mem_size: usize) -> Self {
- let params = ggml_sys::ggml_init_params {
- mem_size,
- mem_buffer: std::ptr::null_mut(),
- no_alloc: false,
- };
-
- let ctx = unsafe { ggml_sys::ggml_init(params) };
-
- Self {
- ctx: NonNull::new(ctx).unwrap(),
- }
- }
-
- pub(crate) fn as_ptr(&self) -> *mut ggml_sys::ggml_context {
- self.ctx.as_ptr()
- }
-}
-
-impl Drop for InnerContext {
- fn drop(&mut self) {
- unsafe {
- ggml_sys::ggml_free(self.ctx.as_ptr());
- }
- }
-}
-
-struct InnerGraph {
- ptr: NonNull<ggml_sys::ggml_cgraph>,
-}
-
-impl InnerGraph {
- fn new(ctx: &InnerContext) -> Self {
- let ptr = unsafe { ggml_sys::ggml_new_graph(ctx.as_ptr()) };
-
- Self {
- ptr: NonNull::new(ptr).unwrap(),
- }
- }
-
- fn as_ptr(&self) -> *mut ggml_sys::ggml_cgraph {
- self.ptr.as_ptr()
- }
-}
-
-pub struct Graph<const OUT_DIMS: usize>
-where
- Dims<OUT_DIMS>: ValidDims,
-{
- inner: InnerGraph,
- out: Tensor<OUT_DIMS>,
-}
-
-impl<const OUT_DIMS: usize> Graph<OUT_DIMS>
-where
- Dims<OUT_DIMS>: ValidDims,
-{
- fn inner(&self) -> &InnerGraph {
- &self.inner
- }
-
- pub fn build(ctx: &Context, out: Tensor<OUT_DIMS>) -> Self {
- let inner = InnerGraph::new(ctx.inner_ctx());
-
- unsafe {
- ggml_sys::ggml_build_forward_expand(inner.as_ptr(), out.as_ptr());
- }
-
- Self { inner, out }
- }
-
- pub fn compute(&mut self, ctx: &Context) {
- ctx.compute(self);
- }
-
- pub fn out(&mut self) -> &Tensor<OUT_DIMS> {
- &self.out
- }
-}
-
-pub struct Context {
- ctx: Arc<InnerContext>,
- n_threads: usize,
-}
-
-impl Context {
- pub fn new(mem_size: usize, n_threads: usize) -> Self {
- let ctx = Arc::new(InnerContext::new(mem_size));
-
- Self { ctx, n_threads }
- }
-
- pub fn build_graph<const OUT_DIMS: usize>(&self, out: Tensor<OUT_DIMS>) -> Graph<OUT_DIMS>
- where
- Dims<OUT_DIMS>: ValidDims,
- {
- Graph::build(self, out)
- }
-
- pub(crate) fn inner_ctx(&self) -> &Arc<InnerContext> {
- &self.ctx
- }
-
- pub fn compute<const OUT_DIMS: usize>(&self, graph: &mut Graph<OUT_DIMS>)
- where
- Dims<OUT_DIMS>: ValidDims,
- {
- unsafe {
- ggml_sys::ggml_graph_compute_with_ctx(
- self.ctx.as_ptr(),
- graph.inner().as_ptr(),
- self.n_threads as i32,
- );
- }
- }
-}
diff --git a/crates/ggml/src/dims.rs b/crates/ggml/src/dims.rs
deleted file mode 100644
index e498527c..00000000
--- a/crates/ggml/src/dims.rs
+++ /dev/null
@@ -1,81 +0,0 @@
-pub struct Dims<const DIMS: usize>
-where
- Self: ValidDims, {}
-
-pub trait ValidDims {}
-
-macro_rules! valid_dims_impl {
- ($($dims:literal)*) => {
- $(
- impl ValidDims for Dims<$dims> {}
- )*
- };
-}
-
-valid_dims_impl!(1 2 3 4);
-
-pub trait DimsGt<const LHS: usize> {}
-
-macro_rules! dims_gt_impl {
- ($($lhs:literal, $rhs:literal)*) => {
- $(
- impl DimsGt<$lhs> for Dims<$rhs>
- where
- Dims<$rhs>: ValidDims,
- {}
- )*
- };
-}
-
-dims_gt_impl!(1, 2);
-dims_gt_impl!(1, 3);
-dims_gt_impl!(1, 4);
-dims_gt_impl!(2, 2);
-dims_gt_impl!(2, 3);
-dims_gt_impl!(2, 4);
-dims_gt_impl!(3, 4);
-
-pub trait DimsLt<const LHS: usize> {}
-
-macro_rules! dims_lt_impl {
- ($($lhs:literal, $rhs:literal)*) => {
- $(
- impl DimsLt<$lhs> for Dims<$rhs>
- where
- Dims<$rhs>: ValidDims,
- {}
- )*
- };
-}
-
-dims_lt_impl!(2, 1);
-dims_lt_impl!(3, 1);
-dims_lt_impl!(4, 1);
-dims_lt_impl!(2, 2);
-dims_lt_impl!(3, 2);
-dims_lt_impl!(4, 2);
-dims_lt_impl!(3, 3);
-dims_lt_impl!(4, 3);
-dims_lt_impl!(4, 4);
-
-pub trait DimsEq<const LHS: usize> {}
-
-impl<const DIMS: usize> DimsEq<DIMS> for Dims<DIMS> where Dims<DIMS>: ValidDims {}
-
-pub trait DimsPlusOne<const N: usize> {}
-
-macro_rules! dims_plus_one_impl {
- ($($n:literal, $res:literal)*) => {
- $(
- impl DimsPlusOne<$n> for Dims<$res>
- where
- Dims<$n>: ValidDims,
- Dims<$res>: ValidDims,
- {}
- )*
- };
-}
-
-dims_plus_one_impl!(1, 2);
-dims_plus_one_impl!(2, 3);
-dims_plus_one_impl!(3, 4);
diff --git a/crates/ggml/src/lib.rs b/crates/ggml/src/lib.rs
deleted file mode 100644
index 2c2f7755..00000000
--- a/crates/ggml/src/lib.rs
+++ /dev/null
@@ -1,8 +0,0 @@
-mod context;
-mod dims;
-pub mod nn;
-pub mod tensor;
-
-pub use context::*;
-pub use dims::*;
-pub use tensor::*;
diff --git a/crates/ggml/src/nn/attn.rs b/crates/ggml/src/nn/attn.rs
deleted file mode 100644
index daf5fc10..00000000
--- a/crates/ggml/src/nn/attn.rs
+++ /dev/null
@@ -1,77 +0,0 @@
-use super::Linear;
-use crate::{Context, Tensor};
-
-pub struct MultiHeadAttention {
- query: Linear,
- key: Linear,
- value: Linear,
- n_embed: u64,
- n_heads: u64,
- head_size: u64,
-}
-impl MultiHeadAttention {
- pub fn new(ctx: &mut Context, n_embed: u64, n_heads: u64) -> Self {
- let head_size = n_embed / n_heads;
- let head_emb = n_heads * head_size;
-
- let query = Linear::new(ctx, n_embed, head_emb);
- let key = Linear::new(ctx, n_embed, head_emb);
- let value = Linear::new(ctx, n_embed, head_emb);
-
- Self {
- query,
- key,
- value,
- n_embed,
- n_heads,
- head_size,
- }
- }
-
- pub fn forward(&self, input: &Tensor<3>) -> Tensor<3> {
- let toks = input.shape()[1];
- let batch_size = input.shape()[2];
- let head_shape = [self.head_size, self.n_heads, toks, batch_size];
-
- let query = self
- .query
- .forward(input)
- .reshape(head_shape)
- .permute([0, 2, 1, 3]);
-
- let key = self
- .key
- .forward(input)
- .reshape(head_shape)
- .permute([0, 2, 1, 3]);
-
- let value = self
- .value
- .forward(input)
- .reshape(head_shape)
- .permute([0, 2, 1, 3]);
-
- let kq = (key * query)
- .scale(1.0 / (self.n_embed as f32).sqrt())
- .softmax();
-
- let kqv = value.permute([1, 0, 2, 3]).contiguous() * kq;
- let out_dims = [self.n_heads * self.head_size, toks, batch_size];
-
- kqv.contiguous().reshape(out_dims)
- }
-}
-#[cfg(test)]
-mod tests {
- use super::*;
- use crate::GgmlType;
-
- #[test]
- fn test_attention() {
- let mut ctx = Context::new(128 * 1024 * 1024, 1);
- let input = Tensor::new(&mut ctx, GgmlType::default(), [128, 3, 5]);
- let attention = MultiHeadAttention::new(&mut ctx, 128, 12);
- let output = attention.forward(&input);
- assert_eq!(output.shape(), [120, 3, 5]);
- }
-}
diff --git a/crates/ggml/src/nn/embedding.rs b/crates/ggml/src/nn/embedding.rs
deleted file mode 100644
index 5fece377..00000000
--- a/crates/ggml/src/nn/embedding.rs
+++ /dev/null
@@ -1,35 +0,0 @@
-use crate::{Context, GgmlType, Tensor};
-
-pub struct Embedding {
- tensor: Tensor<2>,
-}
-
-impl Embedding {
- pub fn new(ctx: &mut Context, vocab_size: u64, embedding_dim: u64) -> Self {
- let tensor = Tensor::new(ctx, GgmlType::default(), [embedding_dim, vocab_size]);
-
- Self { tensor }
- }
-
- pub fn forward(&mut self, input: &Tensor<1>) -> Tensor<2> {
- self.tensor.get_rows(input)
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn test_embedding() {
- let mut ctx = Context::new(128 * 1024 * 1024, 1);
-
- let mut embedding = Embedding::new(&mut ctx, 10, 128);
-
- let input = Tensor::new(&mut ctx, GgmlType::I32, [10]);
-
- let output = embedding.forward(&input);
-
- assert_eq!(output.shape(), [128, 10]);
- }
-}
diff --git a/crates/ggml/src/nn/layer_norm.rs b/crates/ggml/src/nn/layer_norm.rs
deleted file mode 100644
index acc2a81d..00000000
--- a/crates/ggml/src/nn/layer_norm.rs
+++ /dev/null
@@ -1,18 +0,0 @@
-use crate::{Dims, Tensor, ValidDims};
-
-pub struct LayerNorm {
- eps: f32,
-}
-
-impl LayerNorm {
- pub fn new(eps: f32) -> Self {
- Self { eps }
- }
-
- pub fn forward<const DIMS: usize>(&mut self, input: &Tensor<DIMS>) -> Tensor<DIMS>
- where
- Dims<DIMS>: ValidDims,
- {
- input.norm(self.eps)
- }
-}
diff --git a/crates/ggml/src/nn/linear.rs b/crates/ggml/src/nn/linear.rs
deleted file mode 100644
index a7cabc61..00000000
--- a/crates/ggml/src/nn/linear.rs
+++ /dev/null
@@ -1,38 +0,0 @@
-use crate::{Context, GgmlType, Tensor};
-
-pub struct Linear {
- weight: Tensor<3>,
- bias: Option<Tensor<2>>,
-}
-
-impl Linear {
- pub fn new(ctx: &mut Context, input_size: u64, output_size: u64) -> Self {
- let weight = Tensor::new(ctx, GgmlType::default(), [input_size, output_size, 1]);
-
- Self { weight, bias: None }
- }
-
- pub fn forward(&self, input: &Tensor<3>) -> Tensor<3> {
- let res = &self.weight * input;
-
- if let Some(bias) = &self.bias {
- &res + &bias.repeat(&res)
- } else {
- res
- }
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn test_linear() {
- let mut ctx = Context::new(128 * 1024 * 1024, 1);
- let input = Tensor::new(&mut ctx, GgmlType::default(), [128, 3, 5]);
- let linear = Linear::new(&mut ctx, 128, 256);
- let output = linear.forward(&input);
- assert_eq!(output.shape(), [256, 3, 5]);
- }
-}
diff --git a/crates/ggml/src/nn/mod.rs b/crates/ggml/src/nn/mod.rs
deleted file mode 100644
index 72cb3ee2..00000000
--- a/crates/ggml/src/nn/mod.rs
+++ /dev/null
@@ -1,9 +0,0 @@
-mod attn;
-mod embedding;
-mod layer_norm;
-mod linear;
-
-pub use attn::*;
-pub use embedding::*;
-pub use layer_norm::*;
-pub use linear::*;
diff --git a/crates/ggml/src/tensor.rs b/crates/ggml/src/tensor.rs
deleted file mode 100644
index b30884a6..00000000
--- a/crates/ggml/src/tensor.rs
+++ /dev/null
@@ -1,535 +0,0 @@
-use std::ptr::NonNull;
-use std::sync::Arc;
-
-use crate::context::InnerContext;
-use crate::{Context, Dims, DimsGt, DimsPlusOne, ValidDims};
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
-#[allow(non_camel_case_types)]
-pub enum GgmlType {
- #[default]
- F32,
- F16,
- Q4_0,
- Q4_1,
- Q5_0,
- Q5_1,
- Q8_0,
- Q8_1,
- Q2_K,
- Q3_K,
- Q4_K,
- Q5_K,
- Q6_K,
- Q8_K,
- IQ2_XXS,
- IQ2_XS,
- I8,
- I16,
- I32,
- COUNT,
-}
-
-impl GgmlType {
- fn as_raw(&self) -> u32 {
- match &self {
- GgmlType::F32 => 0,
- GgmlType::F16 => 1,
- GgmlType::Q4_0 => 2,
- GgmlType::Q4_1 => 3,
- GgmlType::Q5_0 => 6,
- GgmlType::Q5_1 => 7,
- GgmlType::Q8_0 => 8,
- GgmlType::Q8_1 => 9,
- GgmlType::Q2_K => 10,
- GgmlType::Q3_K => 11,
- GgmlType::Q4_K => 12,
- GgmlType::Q5_K => 13,
- GgmlType::Q6_K => 14,
- GgmlType::Q8_K => 15,
- GgmlType::IQ2_XXS => 16,
- GgmlType::IQ2_XS => 17,
- GgmlType::I8 => 18,
- GgmlType::I16 => 19,
- GgmlType::I32 => 20,
- GgmlType::COUNT => 21,
- }
- }
-}
-
-pub struct Tensor<const DIMS: usize>
-where
- Dims<DIMS>: ValidDims,
-{
- ctx: Arc<InnerContext>,
- type_: GgmlType,
- ptr: NonNull<ggml_sys::ggml_tensor>,
-}
-
-impl<const DIMS: usize> Tensor<DIMS>
-where
- Dims<DIMS>: ValidDims,
-{
- pub fn new(ctx: &mut Context, t: GgmlType, shape: [u64; DIMS]) -> Self {
- let ptr = match DIMS {
- 1 => unsafe {
- ggml_sys::ggml_new_tensor_1d(
- ctx.inner_ctx().as_ptr(),
- t.as_raw() as u32,
- shape[0] as i64,
- )
- },
- 2 => unsafe {
- ggml_sys::ggml_new_tensor_2d(
- ctx.inner_ctx().as_ptr(),
- t.as_raw() as u32,
- shape[0] as i64,
- shape[1] as i64,
- )
- },
- 3 => unsafe {
- ggml_sys::ggml_new_tensor_3d(
- ctx.inner_ctx().as_ptr(),
- t.as_raw() as u32,
- shape[0] as i64,
- shape[1] as i64,
- shape[2] as i64,
- )
- },
- 4 => unsafe {
- ggml_sys::ggml_new_tensor_4d(
- ctx.inner_ctx().as_ptr(),
- t.as_raw() as u32,
- shape[0] as i64,
- shape[1] as i64,
- shape[2] as i64,
- shape[3] as i64,
- )
- },
- _ => unreachable!("{} is not a valid dimension", DIMS),
- };
-
- Self {
- ctx: Arc::clone(ctx.inner_ctx()),
- type_: t,
- ptr: NonNull::new(ptr).unwrap(),
- }
- }
-
- pub(crate) fn as_ptr(&self) -> *mut ggml_sys::ggml_tensor {
- self.ptr.as_ptr()
- }
-
- pub fn num_bytes(&self) -> usize {
- unsafe { ggml_sys::ggml_nbytes(self.ptr.as_ptr()) }
- }
-
- pub fn copy_from_slice(&mut self, slice: &[f32]) {
- if slice.len() as u64 != self.num_elements() {
- panic!("slice length does not match tensor shape");
- }
-
- if self.type_ != GgmlType::F32 {
- panic!("tensor type does not match slice type");
- }
-
- unsafe {
- let ptr = self.ptr.as_ptr().as_ref().unwrap().data;
- (ptr as *mut f32).copy_from_nonoverlapping(slice.as_ptr(), slice.len());
- }
- }
-
- pub fn copy_to_slice(&self, out: &mut [f32]) {
- if out.len() as u64 != self.num_elements() {
- panic!("slice length does not match tensor shape");
- }
-
- if self.type_ != GgmlType::F32 {
- panic!("tensor type does not match slice type");
- }
-
- let data = unsafe {
- std::slice::from_raw_parts(
- self.ptr.as_ptr().as_ref().unwrap().data as *const f32,
- self.num_elements() as usize,
- )
- };
-
- out.copy_from_slice(data);
- }
-
- pub fn num_elements(&self) -> u64 {
- self.shape().into_iter().product()
- }
-
- pub fn shape(&self) -> [u64; DIMS] {
- let shape = unsafe { self.ptr.as_ptr().as_ref().unwrap().ne };
-
- let mut out = [0; DIMS];
-
- for i in 0..DIMS {
- out[i] = shape[i] as u64;
- }
-
- out
- }
-
- pub fn get_rows<const ROW_DIMS: usize, const OUT_DIMS: usize>(
- &self,
- rows: &Tensor<ROW_DIMS>,
- ) -> Tensor<OUT_DIMS>
- where
- Dims<ROW_DIMS>: ValidDims,
- Dims<OUT_DIMS>: DimsPlusOne<ROW_DIMS> + ValidDims,
- {
- debug_assert_eq!(rows.type_, GgmlType::I32, "rows tensor must be of type I32");
-
- let ptr = unsafe {
- ggml_sys::ggml_get_rows(self.ctx.as_ptr(), self.ptr.as_ptr(), rows.ptr.as_ptr())
- };
-
- Tensor {
- ctx: Arc::clone(&self.ctx),
- type_: self.type_,
- ptr: NonNull::new(ptr).unwrap(),
- }
- }
-
- pub fn repeat<const OUT_DIMS: usize>(&self, rep: &Tensor<OUT_DIMS>) -> Tensor<OUT_DIMS>
- where
- Dims<OUT_DIMS>: ValidDims,
- Dims<OUT_DIMS>: DimsGt<DIMS>,
- {
- let ptr =
- unsafe { ggml_sys::ggml_repeat(self.ctx.as_ptr(), self.ptr.as_ptr(), rep.as_ptr()) };
-
- Tensor {
- ctx: Arc::clone(&self.ctx),
- type_: self.type_,
- ptr: NonNull::new(ptr).unwrap(),
- }
- }
-
- pub fn reshape<const OUT_DIMS: usize>(&self, shape: [u64; OUT_DIMS]) -> Tensor<OUT_DIMS>
- where
- Dims<OUT_DIMS>: ValidDims,
- {
- debug_assert_eq!(
- shape.into_iter().product::<u64>(),
- self.num_elements(),
- "reshape must not change the number of elements"
- );
-
- let ptr = match OUT_DIMS {
- 1 => unsafe {
- ggml_sys::ggml_reshape_1d(self.ctx.as_ptr(), self.ptr.as_ptr(), shape[0] as i64)
- },
- 2 => unsafe {
- ggml_sys::ggml_reshape_2d(
- self.ctx.as_ptr(),
- self.ptr.as_ptr(),
- shape[0] as i64,
- shape[1] as i64,
- )
- },
- 3 => unsafe {
- ggml_sys::ggml_reshape_3d(
- self.ctx.as_ptr(),
- self.ptr.as_ptr(),
- shape[0] as i64,
- shape[1] as i64,
- shape[2] as i64,
- )
- },
- 4 => unsafe {
- ggml_sys::ggml_reshape_4d(
- self.ctx.as_ptr(),
- self.ptr.as_ptr(),
- shape[0] as i64,
- shape[1] as i64,
- shape[2] as i64,
- shape[3] as i64,
- )
- },
- _ => unreachable!("{} is not a valid dimension", DIMS),
- };
-
- Tensor {
- ctx: Arc::clone(&self.ctx),
- type_: self.type_,
- ptr: NonNull::new(ptr).unwrap(),
- }
- }
-
- pub fn permute<const OUT_DIMS: usize>(&self, shape: [u64; OUT_DIMS]) -> Tensor<OUT_DIMS>
- where
- Dims<OUT_DIMS>: ValidDims,
- {
- let ptr = match OUT_DIMS {
- 1 => unsafe {
- ggml_sys::ggml_permute(
- self.ctx.as_ptr(),
- self.ptr.as_ptr(),
- shape[0] as i32,
- 1,
- 2,
- 3,
- )
- },
- 2 => unsafe {
- ggml_sys::ggml_permute(
- self.ctx.as_ptr(),
- self.ptr.as_ptr(),
- shape[0] as i32,
- shape[1] as i32,
- 2,
- 3,
- )
- },
- 3 => unsafe {
- ggml_sys::ggml_permute(
- self.ctx.as_ptr(),
- self.ptr.as_ptr(),
- shape[0] as i32,
- shape[1] as i32,
- shape[2] as i32,
- 3,
- )
- },
- 4 => unsafe {
- ggml_sys::ggml_permute(
- self.ctx.as_ptr(),
- self.ptr.as_ptr(),
- shape[0] as i32,
- shape[1] as i32,
- shape[2] as i32,
- shape[3] as i32,
- )
- },
- _ => unreachable!("{} is not a valid dimension", DIMS),
- };
-
- Tensor {
- ctx: Arc::clone(&self.ctx),
- type_: self.type_,
- ptr: NonNull::new(ptr).unwrap(),
- }
- }
-
- pub fn softmax(&self) -> Tensor<DIMS> {
- let ptr = unsafe { ggml_sys::ggml_soft_max(self.ctx.as_ptr(), self.ptr.as_ptr()) };
-
- Tensor {
- ctx: Arc::clone(&self.ctx),
- type_: self.type_,
- ptr: NonNull::new(ptr).unwrap(),
- }
- }
-
- pub fn scale(&self, scale: f32) -> Tensor<DIMS> {
- let ptr = unsafe { ggml_sys::ggml_scale(self.ctx.as_ptr(), self.ptr.as_ptr(), scale) };
-
- Tensor {
- ctx: Arc::clone(&self.ctx),
- type_: self.type_,
- ptr: NonNull::new(ptr).unwrap(),
- }
- }
-
- pub fn contiguous(&self) -> Tensor<DIMS> {
- let ptr = unsafe { ggml_sys::ggml_cont(self.ctx.as_ptr(), self.ptr.as_ptr()) };
-
- Tensor {
- ctx: Arc::clone(&self.ctx),
- type_: self.type_,
- ptr: NonNull::new(ptr).unwrap(),
- }
- }
-
- pub fn as_type(&self, type_: GgmlType) -> Tensor<DIMS> {
- let ptr =
- unsafe { ggml_sys::ggml_cast(self.ctx.as_ptr(), self.ptr.as_ptr(), type_.as_raw()) };
-
- Tensor {
- ctx: Arc::clone(&self.ctx),
- type_,
- ptr: NonNull::new(ptr).unwrap(),
- }
- }
-
- pub fn norm(&self, eps: f32) -> Tensor<DIMS> {
- let ptr = unsafe { ggml_sys::ggml_norm(self.ctx.as_ptr(), self.ptr.as_ptr(), eps) };
-
- Tensor {
- ctx: Arc::clone(&self.ctx),
- type_: self.type_,
- ptr: NonNull::new(ptr).unwrap(),
- }
- }
-
- pub fn load_bytes(&mut self, bytes: &[u8]) {
- debug_assert_eq!(
- bytes.len(),
- self.num_bytes(),
- "bytes must be the same size as the tensor"
- );
-
- unsafe {
- debug_assert_ne!(
- self.as_ptr().as_mut().unwrap().data,
- std::ptr::null_mut(),
- "tensor must be allocated"
- );
-
- (self.as_ptr().as_mut().unwrap().data as *mut u8)
- .copy_from_nonoverlapping(bytes.as_ptr(), self.num_bytes());
- }
- }
-}
-
-macro_rules! std_ops_impl {
- ($trait:ident, $std_func:ident, $func:ident) => {
- impl<const DIMS: usize> std::ops::$trait for Tensor<DIMS>
- where
- Dims<DIMS>: ValidDims,
- {
- type Output = Tensor<DIMS>;
- fn $std_func(self, rhs: Self) -> Self::Output {
- if self.type_ != rhs.type_ {
- panic!("tensor types do not match");
- }
-
- let ptr = unsafe {
- ggml_sys::$func(self.ctx.as_ptr(), self.ptr.as_ptr(), rhs.ptr.as_ptr())
- };
-
- Self::Output {
- ctx: Arc::clone(&self.ctx),
- type_: self.type_,
- ptr: NonNull::new(ptr).unwrap(),
- }
- }
- }
-
- impl<'a, const DIMS: usize> std::ops::$trait<Tensor<DIMS>> for &'a Tensor<DIMS>
- where
- Dims<DIMS>: ValidDims,
- {
- type Output = Tensor<DIMS>;
- fn $std_func(self, rhs: Tensor<DIMS>) -> Self::Output {
- if self.type_ != rhs.type_ {
- panic!("tensor types do not match");
- }
-
- let ptr = unsafe {
- ggml_sys::$func(self.ctx.as_ptr(), self.ptr.as_ptr(), rhs.ptr.as_ptr())
- };
-
- Self::Output {
- ctx: Arc::clone(&self.ctx),
- type_: self.type_,
- ptr: NonNull::new(ptr).unwrap(),
- }
- }
- }
-
- impl<'b, const DIMS: usize> std::ops::$trait<&'b Tensor<DIMS>> for Tensor<DIMS>
- where
- Dims<DIMS>: ValidDims,
- {
- type Output = Tensor<DIMS>;
- fn $std_func(self, rhs: &'b Tensor<DIMS>) -> Self::Output {
- if self.type_ != rhs.type_ {
- panic!("tensor types do not match");
- }
-
- let ptr = unsafe {
- ggml_sys::$func(self.ctx.as_ptr(), self.ptr.as_ptr(), rhs.ptr.as_ptr())
- };
-
- Self::Output {
- ctx: Arc::clone(&self.ctx),
- type_: self.type_,
- ptr: NonNull::new(ptr).unwrap(),
- }
- }
- }
-
- impl<'a, 'b, const DIMS: usize> std::ops::$trait<&'b Tensor<DIMS>> for &'a Tensor<DIMS>
- where
- Dims<DIMS>: ValidDims,
- {
- type Output = Tensor<DIMS>;
- fn $std_func(self, rhs: &'b Tensor<DIMS>) -> Self::Output {
- if self.type_ != rhs.type_ {
- panic!("tensor types do not match");
- }
-
- let ptr = unsafe {
- ggml_sys::$func(self.ctx.as_ptr(), self.ptr.as_ptr(), rhs.ptr.as_ptr())
- };
-
- Self::Output {
- ctx: Arc::clone(&self.ctx),
- type_: self.type_,
- ptr: NonNull::new(ptr).unwrap(),
- }
- }
- }
- };
-}
-
-std_ops_impl!(Add, add, ggml_add);
-std_ops_impl!(Sub, sub, ggml_sub);
-std_ops_impl!(Mul, mul, ggml_mul_mat);
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use crate::{Context, Graph};
- use std::ops::*;
-
- macro_rules! test_1d {
- ($name:ident, $ops:ident, $a:expr, $b:expr, $res:expr) => {
- #[test]
- fn $name() {
- let mut ctx = Context::new(128 * 1024 * 1024, 1);
-
- let mut at = Tensor::new(&mut ctx, GgmlType::F32, [$a.len() as u64]);
- at.copy_from_slice(&$a);
-
- let mut bt = Tensor::new(&mut ctx, GgmlType::F32, [$b.len() as u64]);
- bt.copy_from_slice(&$b);
-
- let ct = at.$ops(bt);
-
- let mut graph = Graph::build(&ctx, ct);
-
- graph.compute(&ctx);
-
- let mut out = vec![0.0; $res.len()];
- graph.out().copy_to_slice(&mut out);
-
- assert_eq!(out, $res);
- }
- };
- }
-
- test_1d!(add_1d, add, [1.0, 2.0], [2.0, 2.0], [3.0, 4.0]);
- test_1d!(mul_1d, mul, [2.0, 5.0], [2.0, 3.0], [19.0]);
- test_1d!(sub_1d, sub, [3.0, 10.0], [2.0, 4.0], [1.0, 6.0]);
-
- #[test]
- fn num_bytes() {
- let mut ctx = Context::new(128 * 1024 * 1024, 1);
- let at = Tensor::new(&mut ctx, GgmlType::F32, [2, 3, 4]);
- assert_eq!(at.num_bytes(), 32 * 2 * 3 * 4 / 8);
- }
-
- #[test]
- fn load_bytes() {
- let mut ctx = Context::new(128 * 1024 * 1024, 1);
- let mut at = Tensor::new(&mut ctx, GgmlType::F32, [2]);
- at.load_bytes(&[1, 2, 3, 4, 5, 6, 7, 8]);
- }
-}
diff --git a/scripts/convert_hf_to_gguf.py b/scripts/convert_hf_to_gguf.py
deleted file mode 100755
index 63eea815..00000000
--- a/scripts/convert_hf_to_gguf.py
+++ /dev/null
@@ -1,170 +0,0 @@
-#!/usr/bin/env python3
-
-from __future__ import annotations
-
-import argparse
-import contextlib
-import json
-import sys
-from pathlib import Path
-from typing import Any, ContextManager, Iterator, cast
-
-import numpy as np
-import torch
-
-import gguf
-
-class Model:
- def __init__(self, dir_model: Path, ftype: int, fname_out: Path):
- self.dir_model = dir_model
- self.ftype = ftype
- self.fname_out = fname_out
- self.is_safetensors = self._is_model_safetensors()
- self.model_name = "model.safetensors" if self.is_safetensors else "pytorch_model.bin"
- self.hparams = Model.load_hparams(self.dir_model)
- self.gguf_writer = gguf.GGUFWriter(fname_out, gguf.MODEL_ARCH_NAMES[gguf.MODEL_ARCH.BERT], endianess=gguf.GGUFEndian.LITTLE, use_temp_file=False)
-
- def get_tensors(self) -> Iterator[tuple[str, torch.Tensor]]:
- ctx: ContextManager[Any]
- if self.is_safetensors:
- from safetensors import safe_open
- ctx = cast(ContextManager[Any], safe_open(self.dir_model / self.model_name, framework="pt", device="cpu"))
- else:
- ctx = contextlib.nullcontext(torch.load(str(self.dir_model / self.model_name), map_location="cpu", weights_only=True))
-
- with ctx as model_part:
- for name in model_part.keys():
- data = model_part.get_tensor(name) if self.is_safetensors else model_part[name]
- yield name, data
-
- def set_gguf_parameters(self):
- self.gguf_writer.add_name(self.dir_model.name)
- self.gguf_writer.add_block_count(self.hparams.get(
- "n_layers", self.hparams.get("num_hidden_layers", self.hparams.get("n_layer")),
- ))
- if (n_ctx := self.hparams.get("max_position_embeddings")) is not None:
- self.gguf_writer.add_context_length(n_ctx)
- if (n_embd := self.hparams.get("hidden_size")) is not None:
- self.gguf_writer.add_embedding_length(n_embd)
- if (n_ff := self.hparams.get("intermediate_size")) is not None:
- self.gguf_writer.add_feed_forward_length(n_ff)
- if (n_head := self.hparams.get("num_attention_heads")) is not None:
- self.gguf_writer.add_head_count(n_head)
- if (n_head_kv := self.hparams.get("num_key_value_heads")) is not None:
- self.gguf_writer.add_head_count_kv(n_head_kv)
-
- if (n_rms_eps := self.hparams.get("rms_norm_eps")) is not None:
- self.gguf_writer.add_layer_norm_rms_eps(n_rms_eps)
- if (n_experts := self.hparams.get("num_local_experts")) is not None:
- self.gguf_writer.add_expert_count(n_experts)
- if (n_experts_used := self.hparams.get("num_experts_per_tok")) is not None:
- self.gguf_writer.add_expert_used_count(n_experts_used)
-
- self.gguf_writer.add_parallel_residual(self.hparams.get("use_parallel_residual", True))
-
- def write_tensors(self):
- for name, data_torch in self.get_tensors():
- # we don't need these
- if name.endswith((".attention.masked_bias", ".attention.bias", ".attention.rotary_emb.inv_freq")):
- continue
-
- old_dtype = data_torch.dtype
-
- # convert any unsupported data types to float32
- if data_torch.dtype not in (torch.float16, torch.float32):
- data_torch = data_torch.to(torch.float32)
-
- data = data_torch.squeeze().numpy()
-
- n_dims = len(data.shape)
- data_dtype = data.dtype
-
- # if f32 desired, convert any float16 to float32
- if self.ftype == 0 and data_dtype == np.float16:
- data = data.astype(np.float32)
-
- # TODO: Why cant we use these float16 as-is? There should be not reason to store float16 as float32
- if self.ftype == 1 and data_dtype == np.float16 and n_dims == 1:
- data = data.astype(np.float32)
-
- # if f16 desired, convert any float32 2-dim weight tensors to float16
- if self.ftype == 1 and data_dtype == np.float32 and name.endswith(".weight") and n_dims == 2:
- data = data.astype(np.float16)
-
- print(f"{name}, n_dims = {n_dims}, {old_dtype} --> {data.dtype}")
-
- self.gguf_writer.add_tensor(name, data)
-
- def write(self):
- self.write_tensors()
- self.gguf_writer.write_header_to_file()
- self.gguf_writer.write_kv_data_to_file()
- self.gguf_writer.write_tensors_to_file()
- self.gguf_writer.close()
-
- @staticmethod
- def load_hparams(dir_model):
- with open(dir_model / "config.json", "r", encoding="utf-8") as f:
- return json.load(f)
-
- def _is_model_safetensors(self) -> bool:
- return (self.dir_model / "model.safetensors").exists()
-
-
-def parse_args() -> argparse.Namespace:
- parser = argparse.ArgumentParser(
- description="Convert a huggingface model to a GGML compatible file")
- parser.add_argument(
- "--outtype", type=str, choices=["f32", "f16"], default="f16",
- help="output format - use f32 for float32, f16 for float16",
- )
- parser.add_argument(
- "model", type=Path,
- help="directory containing model file",
- )
- parser.add_argument(
- "outfile", type=Path,
- help="path to write to",
- )
-
- return parser.parse_args()
-
-
-def main() -> None:
- args = parse_args()
-
- dir_model = args.model
-
- if not dir_model.is_dir():
- print(f'Error: {args.model} is not a directory', file=sys.stderr)
- sys.exit(1)
-
- ftype_map = {
- "f32": gguf.GGMLQuantizationType.F32,
- "f16": gguf.GGMLQuantizationType.F16,
- }
-
- if args.outfile is not None:
- fname_out = args.outfile
- else:
- # output in the same directory as the model by default
- fname_out = dir_model / f'ggml-model-{args.outtype}.gguf'
-
- print(f"Loading model: {dir_model.name}")
-
- hparams = Model.load_hparams(dir_model)
-
- with torch.inference_mode():
- model_instance = Model(dir_model, ftype_map[args.outtype], fname_out)
-
- print("Set model parameters")
- model_instance.set_gguf_parameters()
-
- print(f"Exporting model to '{fname_out}'")
- model_instance.write()
-
- print(f"Model successfully exported to '{fname_out}'")
-
-
-if __name__ == '__main__':
- main()
diff --git a/scripts/download_hf.py b/scripts/download_hf.py
deleted file mode 100644
index 114ee146..00000000
--- a/scripts/download_hf.py
+++ /dev/null
@@ -1,20 +0,0 @@
-from huggingface_hub import snapshot_download
-import argparse
-
-parser = argparse.ArgumentParser(description="Download a huggingface model to a local directory")
-
-parser.add_argument(
- "local_dir", type=str
-)
-
-parser.add_argument(
- "repo_id", type=str
-)
-
-args = parser.parse_args()
-
-snapshot_download(
- repo_id=args.repo_id,
- local_dir=args.local_dir,
- local_dir_use_symlinks=False,
- revision="main")
diff --git a/scripts/export_crossencoder b/scripts/export_crossencoder
index d372a82b..efe1fea4 100755
--- a/scripts/export_crossencoder
+++ b/scripts/export_crossencoder
@@ -1,8 +1,7 @@
#!.venv/bin/python3
import os
-import transformers
-import torch
import sys
+import transformers
MODEL = "cross-encoder/ms-marco-TinyBERT-L-2-v2"
@@ -12,40 +11,8 @@ if os.path.exists("data/cross_encoder/"):
os.system("mkdir data/cross_encoder")
-
model = transformers.AutoModelForSequenceClassification.from_pretrained(MODEL)
-
-model.eval()
-for p in model.parameters():
- p.requires_grad_(False)
-
tokenizer = transformers.AutoTokenizer.from_pretrained(MODEL)
-features = tokenizer(
- ["How many people live in Berlin?", "How many people live in Berlin?"],
- [
- "Berlin has a population of 3,520,031 registered inhabitants in an area of 891.82 square kilometers.",
- "New York City is famous for the Metropolitan Museum of Art.",
- ],
- padding="max_length",
- truncation=True,
- return_tensors="pt",
-)
-input_ids = features["input_ids"]
-attention_mask = features["attention_mask"]
-token_type_ids = features["token_type_ids"]
-tokenizer.save_pretrained("data/cross_encoder")
-
-def run_model(input_ids, attention_mask, token_type_ids):
- res = model(
- input_ids=input_ids,
- attention_mask=attention_mask,
- token_type_ids=token_type_ids,
- return_dict=False,
- )[0]
- return res
-
-
-traced_model = torch.jit.trace(run_model, ((input_ids, attention_mask, token_type_ids)))
-
-traced_model.save("data/cross_encoder/model.pt")
+model.save_pretrained("data/cross_encoder")
+tokenizer.save_pretrained("data/cross_encoder")
diff --git a/scripts/export_dual_encoder b/scripts/export_dual_encoder
index 393f5407..92e99264 100755
--- a/scripts/export_dual_encoder
+++ b/scripts/export_dual_encoder
@@ -1,21 +1,10 @@
#!.venv/bin/python3
import os
-import transformers
-import torch
-import torch.nn.functional as F
import sys
+import transformers
-# MODEL = "nreimers/MiniLM-L6-H384-uncased"
MODEL = "sentence-transformers/all-MiniLM-L6-v2"
-question = "Where is amazon rain forest located"
-context = """The Amazon rainforest (Portuguese: Floresta Amazônica or Amazônia; Spanish: Selva Amazónica, Amazonía or usually Amazonia; French: Forêt amazonienne; Dutch: Amazoneregenwoud), also known in English as Amazonia or the Amazon Jungle, is a moist broadleaf forest that covers most of the Amazon basin of South America. This basin encompasses 7,000,000 square kilometres (2,700,000 sq mi), of
-which 5,500,000 square kilometres (2,100,000 sq mi) are covered by the rainforest. This region includes territory belonging to nine nations.
-The majority of the forest is contained within Brazil, with 60% of the rainforest, followed by Peru with 13%, Colombia with 10%, and with
-minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana. States or departments in four nations contain
-"Amazonas" in their names. The Amazon represents over half of the planet's remaining rainforests, and comprises the largest and most biodiverse tract
-of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species."""
-
if os.path.exists("data/summarizer/dual_encoder/"):
print("data/summarizer/dual_encoder/ already exists. Exiting...")
sys.exit()
@@ -23,48 +12,7 @@ if os.path.exists("data/summarizer/dual_encoder/"):
os.system("mkdir data/summarizer/dual_encoder")
model = transformers.AutoModel.from_pretrained(MODEL)
-
-model.eval()
-for p in model.parameters():
- p.requires_grad_(False)
-
tokenizer = transformers.AutoTokenizer.from_pretrained(MODEL)
-tokenizer.save_pretrained("data/summarizer/dual_encoder")
-
-features = tokenizer(
- [question, context],
- padding="max_length",
- truncation=True,
- return_tensors="pt",
-)
-input_ids = features["input_ids"]
-attention_mask = features["attention_mask"]
-token_type_ids = features["token_type_ids"]
-
-def mean_pooling(model_output, attention_mask):
- token_embeddings = model_output[
- 0
- ] # First element of model_output contains all token embeddings
- input_mask_expanded = (
- attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
- )
- return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(
- input_mask_expanded.sum(1), min=1e-9
- )
-
-
-def run_model(input_ids, token_type_ids, attention_mask):
- res = model(
- input_ids=input_ids,
- token_type_ids=token_type_ids,
- attention_mask=attention_mask,
- return_dict=False,
- )
- embs = mean_pooling(res, attention_mask)
- return F.normalize(embs, p=2, dim=1)
-
-
-traced_model = torch.jit.trace(run_model, ((input_ids, token_type_ids, attention_mask)))
-
-traced_model.save("data/summarizer/dual_encoder/model.pt")
+model.save_pretrained("data/summarizer/dual_encoder")
+tokenizer.save_pretrained("data/summarizer/dual_encoder")