web search done right
move neural network models from torch to candle
Mikkel Denkerd7e564d91a21
diff --git a/.gitmodules b/.gitmodulesindex 73941dd1..e9ef195d 100644--- a/.gitmodules+++ b/.gitmodules@@ -1,6 +1,3 @@[submodule "optics/testcases"]path = optics/testcases/samplesurl = https://github.com/StractOrg/sample-optics-[submodule "crates/ggml-sys/ggml-src"]- path = crates/ggml-sys/ggml-src- url = https://github.com/ggerganov/ggmldiff --git a/Cargo.lock b/Cargo.lockindex 102a6d18..1fcd9f3d 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -460,29 +460,6 @@ dependencies = ["syn 2.0.38",]-[[package]]-name = "bindgen"-version = "0.69.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a4c69fae65a523209d34240b60abe0c42d33d1045d445c0839d8a4894a736e2d"-dependencies = [- "bitflags 2.4.1",- "cexpr",- "clang-sys",- "lazy_static",- "lazycell",- "log",- "peeking_take_while",- "prettyplease",- "proc-macro2",- "quote",- "regex",- "rustc-hash",- "shlex",- "syn 2.0.38",- "which",-]-[[package]]name = "bit-set"version = "0.5.3"@@ -643,6 +620,61 @@ dependencies = ["zip",]+[[package]]+name = "candle-core"+version = "0.3.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6db8659ea87ee8197d2fc627348916cce0561330ee7ae3874e771691d3cecb2f"+dependencies = [+ "byteorder 1.5.0",+ "gemm",+ "half 2.3.1",+ "memmap2",+ "num-traits",+ "num_cpus",+ "rand 0.8.5",+ "rand_distr",+ "rayon",+ "safetensors 0.4.2",+ "thiserror",+ "yoke",+ "zip",+]++[[package]]+name = "candle-nn"+version = "0.3.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7ddce8312032760a6791d6adc9c56dc54fd7c1be38d85dcc4862f1c75228bbc7"+dependencies = [+ "candle-core",+ "half 2.3.1",+ "num-traits",+ "rayon",+ "safetensors 0.4.2",+ "serde",+ "thiserror",+]++[[package]]+name = "candle-transformers"+version = "0.3.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "68834a0cacb7e002d1f4abfe26a7cd1237e2ba342fddcf2e30913c4edb96409d"+dependencies = [+ "byteorder 1.5.0",+ "candle-core",+ "candle-nn",+ "num-traits",+ "rand 0.8.5",+ "rayon",+ "serde",+ "serde_json",+ "serde_plain",+ "tracing",+ "wav",+]+[[package]]name = "cast"version = "0.3.0"@@ -799,15 +831,6 @@ version = "0.5.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961"-[[package]]-name = "cmake"-version = "0.1.50"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130"-dependencies = [- "cc",-]-[[package]]name = "color_quant"version = "1.1.0"@@ -1229,6 +1252,16 @@ dependencies = ["dtoa",]+[[package]]+name = "dyn-stack"+version = "0.10.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "56e53799688f5632f364f8fb387488dd05db9fe45db7011be066fc20e7027f8b"+dependencies = [+ "bytemuck",+ "reborrow",+]+[[package]]name = "either"version = "1.9.0"@@ -1259,6 +1292,18 @@ dependencies = ["cfg-if",]+[[package]]+name = "enum-as-inner"+version = "0.6.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5ffccbb6966c05b32ef8fbac435df276c4ae4d3dc55a8cd0eb9745e6c12f546a"+dependencies = [+ "heck 0.4.1",+ "proc-macro2",+ "quote",+ "syn 2.0.38",+]+[[package]]name = "equivalent"version = "1.0.1"@@ -1558,6 +1603,124 @@ dependencies = ["byteorder 1.5.0",]+[[package]]+name = "gemm"+version = "0.17.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e97d506c68f4fb12325b52a638e7d54cc87e3593a4ded0de60218b6dfd65f645"+dependencies = [+ "dyn-stack",+ "gemm-c32",+ "gemm-c64",+ "gemm-common",+ "gemm-f16",+ "gemm-f32",+ "gemm-f64",+ "num-complex",+ "num-traits",+ "paste",+ "raw-cpuid",+ "seq-macro",+]++[[package]]+name = "gemm-c32"+version = "0.17.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0dd16f26e8f34661edc906d8c9522b59ec1655c865a98a58950d0246eeaca9da"+dependencies = [+ "dyn-stack",+ "gemm-common",+ "num-complex",+ "num-traits",+ "paste",+ "raw-cpuid",+ "seq-macro",+]++[[package]]+name = "gemm-c64"+version = "0.17.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a8e34381bc060b47fbd25522a281799ef763cd27f43bbd1783d935774659242a"+dependencies = [+ "dyn-stack",+ "gemm-common",+ "num-complex",+ "num-traits",+ "paste",+ "raw-cpuid",+ "seq-macro",+]++[[package]]+name = "gemm-common"+version = "0.17.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "22518a76339b09276f77c3166c44262e55f633712fe8a44fd0573505887feeab"+dependencies = [+ "bytemuck",+ "dyn-stack",+ "half 2.3.1",+ "num-complex",+ "num-traits",+ "once_cell",+ "paste",+ "pulp",+ "raw-cpuid",+ "rayon",+ "seq-macro",+ "sysctl",+]++[[package]]+name = "gemm-f16"+version = "0.17.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "70409bbf3ef83b38cbe4a58cd4b797c1c27902505bdd926a588ea61b6c550a84"+dependencies = [+ "dyn-stack",+ "gemm-common",+ "gemm-f32",+ "half 2.3.1",+ "num-complex",+ "num-traits",+ "paste",+ "raw-cpuid",+ "rayon",+ "seq-macro",+]++[[package]]+name = "gemm-f32"+version = "0.17.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5ea3068edca27f100964157211782eba19e961aa4d0d2bdac3e1775a51aa7680"+dependencies = [+ "dyn-stack",+ "gemm-common",+ "num-complex",+ "num-traits",+ "paste",+ "raw-cpuid",+ "seq-macro",+]++[[package]]+name = "gemm-f64"+version = "0.17.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5fd41e8f5a60dce8d8acd852a3f4b22f8e18be957e1937731be692c037652510"+dependencies = [+ "dyn-stack",+ "gemm-common",+ "num-complex",+ "num-traits",+ "paste",+ "raw-cpuid",+ "seq-macro",+]+[[package]]name = "generator"version = "0.7.5"@@ -1605,22 +1768,6 @@ dependencies = ["wasm-bindgen",]-[[package]]-name = "ggml"-version = "0.1.0"-dependencies = [- "ggml-sys",-]--[[package]]-name = "ggml-sys"-version = "0.1.0"-dependencies = [- "bindgen 0.69.2",- "cc",- "cmake",-]-[[package]]name = "ghash"version = "0.5.0"@@ -1703,8 +1850,12 @@ version = "2.3.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "bc52e53916c08643f1b56ec082790d1e86a32e58dc5268f897f313fbae7b4872"dependencies = [+ "bytemuck","cfg-if","crunchy",+ "num-traits",+ "rand 0.8.5",+ "rand_distr","serde",]@@ -1770,15 +1921,6 @@ dependencies = ["digest",]-[[package]]-name = "home"-version = "0.5.9"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5"-dependencies = [- "windows-sys 0.52.0",-]-[[package]]name = "html5ever"version = "0.26.0"@@ -2267,9 +2409,9 @@ checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25"[[package]]name = "libc"-version = "0.2.149"+version = "0.2.153"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b"+checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"[[package]]name = "libloading"@@ -2293,7 +2435,7 @@ version = "0.11.0+8.1.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "d3386f101bcb4bd252d8e9d2fb41ec3b0862a15a62b478c355b2982efa469e3e"dependencies = [- "bindgen 0.65.1",+ "bindgen","bzip2-sys","cc","glob",@@ -2551,11 +2693,12 @@ dependencies = [[[package]]name = "memmap2"-version = "0.9.0"+version = "0.9.4"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "deaba38d7abf1d4cca21cc89e932e542ba2b9258664d2a9ef0e61512039c9375"+checksum = "fe751422e4a8caa417e13c3ea66452215d7d63e19e604f4980461212f3ae1322"dependencies = ["libc",+ "stable_deref_trait",][[package]]@@ -2741,6 +2884,7 @@ version = "0.4.4"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "1ba157ca0885411de85d6ca030ba7e2a83a28636056c7c699b07c8b6f7383214"dependencies = [+ "bytemuck","num-traits",]@@ -3363,6 +3507,18 @@ dependencies = ["psl-types",]+[[package]]+name = "pulp"+version = "0.18.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "16785ee69419641c75affff7c9fdbdb7c0ab26dc9a5fb5218c2a2e9e4ef2087d"+dependencies = [+ "bytemuck",+ "libm",+ "num-complex",+ "reborrow",+]+[[package]]name = "qoi"version = "0.4.1"@@ -3475,6 +3631,16 @@ dependencies = ["getrandom 0.2.10",]+[[package]]+name = "rand_distr"+version = "0.4.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31"+dependencies = [+ "num-traits",+ "rand 0.8.5",+]+[[package]]name = "rand_hc"version = "0.2.0"@@ -3511,6 +3677,15 @@ dependencies = ["rand_core 0.6.4",]+[[package]]+name = "raw-cpuid"+version = "10.7.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332"+dependencies = [+ "bitflags 1.3.2",+]+[[package]]name = "rawpointer"version = "0.2.1"@@ -3548,6 +3723,12 @@ dependencies = ["crossbeam-utils",]+[[package]]+name = "reborrow"+version = "0.5.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430"+[[package]]name = "redox_syscall"version = "0.2.16"@@ -3667,6 +3848,12 @@ dependencies = ["winreg",]+[[package]]+name = "riff"+version = "1.0.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b9b1a3d5f46d53f4a3478e2be4a5a5ce5108ea58b100dcd139830eae7f79a3a1"+[[package]]name = "ring"version = "0.17.4"@@ -3888,6 +4075,16 @@ dependencies = ["serde_json",]+[[package]]+name = "safetensors"+version = "0.4.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8d980e6bfb34436fb0a81e42bc41af43f11805bbbca443e7f68e9faaabe669ed"+dependencies = [+ "serde",+ "serde_json",+]+[[package]]name = "same-file"version = "1.0.6"@@ -4030,6 +4227,12 @@ version = "1.0.20"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090"+[[package]]+name = "seq-macro"+version = "0.3.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4"+[[package]]name = "serde"version = "1.0.189"@@ -4071,6 +4274,15 @@ dependencies = ["serde",]+[[package]]+name = "serde_plain"+version = "1.0.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9ce1fc6db65a611022b23a0dec6975d63fb80a302cb3388835ff02c097258d50"+dependencies = [+ "serde",+]+[[package]]name = "serde_spanned"version = "0.6.3"@@ -4286,6 +4498,9 @@ dependencies = ["bytemuck","byteorder 1.5.0","bzip2",+ "candle-core",+ "candle-nn",+ "candle-transformers","chitchat","chrono","clap",@@ -4300,7 +4515,6 @@ dependencies = ["fnv","fst","futures",- "ggml","half 2.3.1","hashbrown 0.14.1","http 1.0.0",@@ -4338,7 +4552,7 @@ dependencies = ["rocksdb","rust-s3","rust-stemmers",- "safetensors",+ "safetensors 0.3.3","scylla","serde","serde_json",@@ -4448,6 +4662,31 @@ version = "0.1.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"+[[package]]+name = "synstructure"+version = "0.13.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 2.0.38",+]++[[package]]+name = "sysctl"+version = "0.5.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ec7dddc5f0fee506baf8b9fdb989e242f17e4b11c61dfbb0635b705217199eea"+dependencies = [+ "bitflags 2.4.1",+ "byteorder 1.5.0",+ "enum-as-inner",+ "libc",+ "thiserror",+ "walkdir",+]+[[package]]name = "system-configuration"version = "0.5.1"@@ -4630,7 +4869,7 @@ dependencies = ["libc","ndarray","rand 0.8.5",- "safetensors",+ "safetensors 0.3.3","thiserror","torch-sys","zip",@@ -5401,6 +5640,15 @@ dependencies = ["web-sys",]+[[package]]+name = "wav"+version = "1.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a65e199c799848b4f997072aa4d673c034f80f40191f97fe2f0a23f410be1609"+dependencies = [+ "riff",+]+[[package]]name = "web-sys"version = "0.3.64"@@ -5428,18 +5676,6 @@ dependencies = ["serde",]-[[package]]-name = "which"-version = "4.4.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7"-dependencies = [- "either",- "home",- "once_cell",- "rustix",-]-[[package]]name = "winapi"version = "0.3.9"@@ -5498,15 +5734,6 @@ dependencies = ["windows-targets 0.48.5",]-[[package]]-name = "windows-sys"-version = "0.52.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"-dependencies = [- "windows-targets 0.52.0",-]-[[package]]name = "windows-targets"version = "0.42.2"@@ -5537,21 +5764,6 @@ dependencies = ["windows_x86_64_msvc 0.48.5",]-[[package]]-name = "windows-targets"-version = "0.52.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd"-dependencies = [- "windows_aarch64_gnullvm 0.52.0",- "windows_aarch64_msvc 0.52.0",- "windows_i686_gnu 0.52.0",- "windows_i686_msvc 0.52.0",- "windows_x86_64_gnu 0.52.0",- "windows_x86_64_gnullvm 0.52.0",- "windows_x86_64_msvc 0.52.0",-]-[[package]]name = "windows_aarch64_gnullvm"version = "0.42.2"@@ -5564,12 +5776,6 @@ version = "0.48.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"-[[package]]-name = "windows_aarch64_gnullvm"-version = "0.52.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea"-[[package]]name = "windows_aarch64_msvc"version = "0.42.2"@@ -5582,12 +5788,6 @@ version = "0.48.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"-[[package]]-name = "windows_aarch64_msvc"-version = "0.52.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef"-[[package]]name = "windows_i686_gnu"version = "0.42.2"@@ -5600,12 +5800,6 @@ version = "0.48.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"-[[package]]-name = "windows_i686_gnu"-version = "0.52.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313"-[[package]]name = "windows_i686_msvc"version = "0.42.2"@@ -5618,12 +5812,6 @@ version = "0.48.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"-[[package]]-name = "windows_i686_msvc"-version = "0.52.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a"-[[package]]name = "windows_x86_64_gnu"version = "0.42.2"@@ -5636,12 +5824,6 @@ version = "0.48.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"-[[package]]-name = "windows_x86_64_gnu"-version = "0.52.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd"-[[package]]name = "windows_x86_64_gnullvm"version = "0.42.2"@@ -5654,12 +5836,6 @@ version = "0.48.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"-[[package]]-name = "windows_x86_64_gnullvm"-version = "0.52.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e"-[[package]]name = "windows_x86_64_msvc"version = "0.42.2"@@ -5672,12 +5848,6 @@ version = "0.48.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"-[[package]]-name = "windows_x86_64_msvc"-version = "0.52.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04"-[[package]]name = "winnow"version = "0.5.17"@@ -5724,6 +5894,51 @@ dependencies = ["linked-hash-map",]+[[package]]+name = "yoke"+version = "0.7.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "65e71b2e4f287f467794c671e2b8f8a5f3716b3c829079a1c44740148eff07e4"+dependencies = [+ "serde",+ "stable_deref_trait",+ "yoke-derive",+ "zerofrom",+]++[[package]]+name = "yoke-derive"+version = "0.7.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9e6936f0cce458098a201c245a11bef556c6a0181129c7034d10d76d1ec3a2b8"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 2.0.38",+ "synstructure",+]++[[package]]+name = "zerofrom"+version = "0.1.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "655b0814c5c0b19ade497851070c640773304939a6c0fd5f5fb43da0696d05b7"+dependencies = [+ "zerofrom-derive",+]++[[package]]+name = "zerofrom-derive"+version = "0.1.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e6a647510471d372f2e6c2e6b7219e44d8c574d24fdc11c610a61455782f18c3"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 2.0.38",+ "synstructure",+]+[[package]]name = "zimba"version = "0.1.0"diff --git a/Cargo.toml b/Cargo.tomlindex 58b26b64..ccdda92b 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -4,7 +4,6 @@ members = ["crates/optics","crates/kuchiki","crates/zimba",- "crates/ggml-sys", "crates/ggml",]resolver = "2"@@ -29,6 +28,9 @@ bitvec = "1.0.1"bytemuck = {version = "1.13.1", features = ["derive"]}byteorder = "1.4.3"bzip2 = "0.4.4"+candle-transformers = "0.3.3"+candle-nn = "0.3.3"+candle-core = "0.3.3"chitchat = "0.5.0"chrono = {version = "0.4.23", features = ["serde"]}clap = {version = "4.4.6", features = ["derive"]}diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.tomlindex 173cc652..402db030 100644--- a/crates/core/Cargo.toml+++ b/crates/core/Cargo.toml@@ -34,6 +34,9 @@ bitvec = {workspace = true}bytemuck = {workspace = true}byteorder = {workspace = true}bzip2 = {workspace = true}+candle-transformers = {workspace = true}+candle-nn = {workspace = true}+candle-core = {workspace = true}chitchat = {workspace = true}chrono = {workspace = true}clap = {workspace = true}@@ -47,7 +50,6 @@ flate2 = {workspace = true}fnv = {workspace = true}fst = {workspace = true}futures = {workspace = true}-ggml = {path = "../ggml"}half = {workspace = true}hashbrown = {workspace = true}http = {workspace = true}diff --git a/crates/core/src/llm_utils.rs b/crates/core/src/llm_utils.rsindex ea87c4bf..60dc04cc 100644--- a/crates/core/src/llm_utils.rs+++ b/crates/core/src/llm_utils.rs@@ -19,20 +19,8 @@ use anyhow::anyhow;use eventsource_stream::Eventsource;use futures::stream::Stream;#[cfg(feature = "libtorch")]-use tch::Tensor;use tokio_stream::StreamExt;-#[cfg(feature = "libtorch")]-pub struct ClonableTensor(pub Tensor);--#[cfg(feature = "libtorch")]-impl Clone for ClonableTensor {- fn clone(&self) -> Self {- let out = Tensor::empty(self.0.size(), (self.0.kind(), self.0.device()));- ClonableTensor(self.0.clone(&out))- }-}-pub struct OpenAiApi {api: String,key: Option<String>,diff --git a/crates/core/src/models/bert.rs b/crates/core/src/models/bert.rsindex d7733cf6..29077f79 100644--- a/crates/core/src/models/bert.rs+++ b/crates/core/src/models/bert.rs@@ -1,44 +1,569 @@-use ggml::{nn, Context, Tensor};+//! this file has been copied from the candle-transformers project and modified+//! to include an optional pooler layer and use an attention mask for padding tokens.+//!+//! The original file can be found at:+//!+//! https://github.com/huggingface/candle/blob/main/candle-examples/examples/bert/main.rs-struct BertEmbeddings {- word_embeddings: nn::Embedding,- position_embeddings: nn::Embedding,- token_type_embeddings: nn::Embedding,- layer_norm: nn::LayerNorm,+use candle_core::{Device, Result, Tensor};+use candle_nn::{embedding, Embedding, Module, VarBuilder};+use candle_transformers::models::with_tracing::{layer_norm, linear, LayerNorm, Linear};+use serde::Deserialize;++#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]+#[serde(rename_all = "lowercase")]+pub enum HiddenAct {+ Gelu,+ GeluApproximate,+ Relu,}-impl BertEmbeddings {- pub fn new(ctx: &mut Context, vocab_size: u64, hidden_size: u64) -> Self {- let word_embeddings = nn::Embedding::new(ctx, vocab_size, hidden_size);- let position_embeddings = nn::Embedding::new(ctx, 512, hidden_size);- let token_type_embeddings = nn::Embedding::new(ctx, 2, hidden_size);- let layer_norm = nn::LayerNorm::new(1e-12);+struct HiddenActLayer {+ act: HiddenAct,+ span: tracing::Span,+}++impl HiddenActLayer {+ fn new(act: HiddenAct) -> Self {+ let span = tracing::span!(tracing::Level::TRACE, "hidden-act");+ Self { act, span }+ }++ fn forward(&self, xs: &Tensor) -> candle_core::Result<Tensor> {+ let _enter = self.span.enter();+ match self.act {+ // https://github.com/huggingface/transformers/blob/cd4584e3c809bb9e1392ccd3fe38b40daba5519a/src/transformers/activations.py#L213+ HiddenAct::Gelu => xs.gelu_erf(),+ HiddenAct::GeluApproximate => xs.gelu(),+ HiddenAct::Relu => xs.relu(),+ }+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Default)]+#[serde(rename_all = "lowercase")]+pub enum PositionEmbeddingType {+ #[default]+ Absolute,+}+// https://github.com/huggingface/transformers/blob/6eedfa6dd15dc1e22a55ae036f681914e5a0d9a1/src/transformers/models/bert/configuration_bert.py#L1+#[derive(Debug, Clone, PartialEq, Deserialize)]+pub struct Config {+ pub vocab_size: usize,+ pub hidden_size: usize,+ pub num_hidden_layers: usize,+ pub num_attention_heads: usize,+ pub intermediate_size: usize,+ pub hidden_act: HiddenAct,+ pub hidden_dropout_prob: f64,+ pub max_position_embeddings: usize,+ pub type_vocab_size: usize,+ pub initializer_range: f64,+ pub layer_norm_eps: f64,+ pub pad_token_id: usize,+ #[serde(default)]+ pub position_embedding_type: PositionEmbeddingType,+ #[serde(default)]+ pub use_cache: bool,+ pub classifier_dropout: Option<f64>,+ pub model_type: Option<String>,+}++impl Default for Config {+ fn default() -> Self {Self {+ vocab_size: 30522,+ hidden_size: 768,+ num_hidden_layers: 12,+ num_attention_heads: 12,+ intermediate_size: 3072,+ hidden_act: HiddenAct::Gelu,+ hidden_dropout_prob: 0.1,+ max_position_embeddings: 512,+ type_vocab_size: 2,+ initializer_range: 0.02,+ layer_norm_eps: 1e-12,+ pad_token_id: 0,+ position_embedding_type: PositionEmbeddingType::Absolute,+ use_cache: true,+ classifier_dropout: None,+ model_type: Some("bert".to_string()),+ }+ }+}++impl Config {+ fn _all_mini_lm_l6_v2() -> Self {+ // https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/blob/main/config.json+ Self {+ vocab_size: 30522,+ hidden_size: 384,+ num_hidden_layers: 6,+ num_attention_heads: 12,+ intermediate_size: 1536,+ hidden_act: HiddenAct::Gelu,+ hidden_dropout_prob: 0.1,+ max_position_embeddings: 512,+ type_vocab_size: 2,+ initializer_range: 0.02,+ layer_norm_eps: 1e-12,+ pad_token_id: 0,+ position_embedding_type: PositionEmbeddingType::Absolute,+ use_cache: true,+ classifier_dropout: None,+ model_type: Some("bert".to_string()),+ }+ }+}++struct Dropout {+ #[allow(dead_code)]+ pr: f64,+}++impl Dropout {+ fn new(pr: f64) -> Self {+ Self { pr }+ }+}++impl Module for Dropout {+ fn forward(&self, x: &Tensor) -> Result<Tensor> {+ // TODO+ Ok(x.clone())+ }+}++// https://github.com/huggingface/transformers/blob/6eedfa6dd15dc1e22a55ae036f681914e5a0d9a1/src/transformers/models/bert/modeling_bert.py#L180+struct BertEmbeddings {+ word_embeddings: Embedding,+ position_embeddings: Option<Embedding>,+ token_type_embeddings: Embedding,+ layer_norm: LayerNorm,+ dropout: Dropout,+ span: tracing::Span,+}++impl BertEmbeddings {+ fn load(vb: VarBuilder, config: &Config) -> Result<Self> {+ let word_embeddings = embedding(+ config.vocab_size,+ config.hidden_size,+ vb.pp("word_embeddings"),+ )?;+ let position_embeddings = embedding(+ config.max_position_embeddings,+ config.hidden_size,+ vb.pp("position_embeddings"),+ )?;+ let token_type_embeddings = embedding(+ config.type_vocab_size,+ config.hidden_size,+ vb.pp("token_type_embeddings"),+ )?;+ let layer_norm = layer_norm(+ config.hidden_size,+ config.layer_norm_eps,+ vb.pp("LayerNorm"),+ )?;+ Ok(Self {word_embeddings,- position_embeddings,+ position_embeddings: Some(position_embeddings),token_type_embeddings,layer_norm,+ dropout: Dropout::new(config.hidden_dropout_prob),+ span: tracing::span!(tracing::Level::TRACE, "embeddings"),+ })+ }++ fn forward(&self, input_ids: &Tensor, token_type_ids: &Tensor) -> Result<Tensor> {+ let _enter = self.span.enter();+ let (_bsize, seq_len) = input_ids.dims2()?;+ let input_embeddings = self.word_embeddings.forward(input_ids)?;+ let token_type_embeddings = self.token_type_embeddings.forward(token_type_ids)?;+ let mut embeddings = (&input_embeddings + token_type_embeddings)?;+ if let Some(position_embeddings) = &self.position_embeddings {+ // TODO: Proper absolute positions?+ let position_ids = (0..seq_len as u32).collect::<Vec<_>>();+ let position_ids = Tensor::new(&position_ids[..], input_ids.device())?;+ embeddings = embeddings.broadcast_add(&position_embeddings.forward(&position_ids)?)?}+ let embeddings = self.layer_norm.forward(&embeddings)?;+ let embeddings = self.dropout.forward(&embeddings)?;+ Ok(embeddings)}+}- pub fn forward(- &mut self,- input_ids: &Tensor<1>,- token_type_ids: &Tensor<1>,- position_ids: &Tensor<1>,- ) -> Tensor<2> {- let word_embeddings = self.word_embeddings.forward(input_ids);- let position_embeddings = self.position_embeddings.forward(position_ids);- let token_type_embeddings = self.token_type_embeddings.forward(token_type_ids);+struct BertSelfAttention {+ query: Linear,+ key: Linear,+ value: Linear,+ dropout: Dropout,+ num_attention_heads: usize,+ attention_head_size: usize,+ span: tracing::Span,+ span_softmax: tracing::Span,+}++impl BertSelfAttention {+ fn load(vb: VarBuilder, config: &Config) -> Result<Self> {+ let attention_head_size = config.hidden_size / config.num_attention_heads;+ let all_head_size = config.num_attention_heads * attention_head_size;+ let dropout = Dropout::new(config.hidden_dropout_prob);+ let hidden_size = config.hidden_size;+ let query = linear(hidden_size, all_head_size, vb.pp("query"))?;+ let value = linear(hidden_size, all_head_size, vb.pp("value"))?;+ let key = linear(hidden_size, all_head_size, vb.pp("key"))?;+ Ok(Self {+ query,+ key,+ value,+ dropout,+ num_attention_heads: config.num_attention_heads,+ attention_head_size,+ span: tracing::span!(tracing::Level::TRACE, "self-attn"),+ span_softmax: tracing::span!(tracing::Level::TRACE, "softmax"),+ })+ }++ fn transpose_for_scores(&self, xs: &Tensor) -> Result<Tensor> {+ let mut new_x_shape = xs.dims().to_vec();+ new_x_shape.pop();+ new_x_shape.push(self.num_attention_heads);+ new_x_shape.push(self.attention_head_size);+ let xs = xs.reshape(new_x_shape.as_slice())?.transpose(1, 2)?;+ xs.contiguous()+ }+}++impl BertSelfAttention {+ pub fn forward(&self, hidden_states: &Tensor, attention_mask: &Tensor) -> Result<Tensor> {+ let _enter = self.span.enter();+ let query_layer = self.query.forward(hidden_states)?;+ let key_layer = self.key.forward(hidden_states)?;+ let value_layer = self.value.forward(hidden_states)?;++ let query_layer = self.transpose_for_scores(&query_layer)?;+ let key_layer = self.transpose_for_scores(&key_layer)?;+ let value_layer = self.transpose_for_scores(&value_layer)?;++ let attention_scores = query_layer.matmul(&key_layer.t()?)?;+ let attention_scores = (attention_scores / (self.attention_head_size as f64).sqrt())?;+ let attention_scores = attention_scores.broadcast_add(&attention_mask)?;++ let attention_probs = {+ let _enter_sm = self.span_softmax.enter();+ candle_nn::ops::softmax(&attention_scores, candle_core::D::Minus1)?+ };+ let attention_probs = self.dropout.forward(&attention_probs)?;++ let context_layer = attention_probs.matmul(&value_layer)?;+ let context_layer = context_layer.transpose(1, 2)?.contiguous()?;+ let context_layer = context_layer.flatten_from(candle_core::D::Minus2)?;+ Ok(context_layer)+ }+}++struct BertSelfOutput {+ dense: Linear,+ layer_norm: LayerNorm,+ dropout: Dropout,+ span: tracing::Span,+}++impl BertSelfOutput {+ fn load(vb: VarBuilder, config: &Config) -> Result<Self> {+ let dense = linear(config.hidden_size, config.hidden_size, vb.pp("dense"))?;+ let layer_norm = layer_norm(+ config.hidden_size,+ config.layer_norm_eps,+ vb.pp("LayerNorm"),+ )?;+ let dropout = Dropout::new(config.hidden_dropout_prob);+ Ok(Self {+ dense,+ layer_norm,+ dropout,+ span: tracing::span!(tracing::Level::TRACE, "self-out"),+ })+ }++ fn forward(&self, hidden_states: &Tensor, input_tensor: &Tensor) -> Result<Tensor> {+ let _enter = self.span.enter();+ let hidden_states = self.dense.forward(hidden_states)?;+ let hidden_states = self.dropout.forward(&hidden_states)?;+ self.layer_norm.forward(&(hidden_states + input_tensor)?)+ }+}++// https://github.com/huggingface/transformers/blob/6eedfa6dd15dc1e22a55ae036f681914e5a0d9a1/src/transformers/models/bert/modeling_bert.py#L392+struct BertAttention {+ self_attention: BertSelfAttention,+ self_output: BertSelfOutput,+ span: tracing::Span,+}- let embeddings = word_embeddings + position_embeddings + token_type_embeddings;+impl BertAttention {+ fn load(vb: VarBuilder, config: &Config) -> Result<Self> {+ let self_attention = BertSelfAttention::load(vb.pp("self"), config)?;+ let self_output = BertSelfOutput::load(vb.pp("output"), config)?;+ Ok(Self {+ self_attention,+ self_output,+ span: tracing::span!(tracing::Level::TRACE, "attn"),+ })+ }+}++impl BertAttention {+ fn forward(&self, hidden_states: &Tensor, attention_mask: &Tensor) -> Result<Tensor> {+ let _enter = self.span.enter();+ let self_outputs = self.self_attention.forward(hidden_states, attention_mask)?;+ let attention_output = self.self_output.forward(&self_outputs, hidden_states)?;+ Ok(attention_output)+ }+}++// https://github.com/huggingface/transformers/blob/6eedfa6dd15dc1e22a55ae036f681914e5a0d9a1/src/transformers/models/bert/modeling_bert.py#L441+struct BertIntermediate {+ dense: Linear,+ intermediate_act: HiddenActLayer,+ span: tracing::Span,+}++impl BertIntermediate {+ fn load(vb: VarBuilder, config: &Config) -> Result<Self> {+ let dense = linear(config.hidden_size, config.intermediate_size, vb.pp("dense"))?;+ Ok(Self {+ dense,+ intermediate_act: HiddenActLayer::new(config.hidden_act),+ span: tracing::span!(tracing::Level::TRACE, "inter"),+ })+ }+}++impl Module for BertIntermediate {+ fn forward(&self, hidden_states: &Tensor) -> Result<Tensor> {+ let _enter = self.span.enter();+ let hidden_states = self.dense.forward(hidden_states)?;+ let ys = self.intermediate_act.forward(&hidden_states)?;+ Ok(ys)+ }+}++// https://github.com/huggingface/transformers/blob/6eedfa6dd15dc1e22a55ae036f681914e5a0d9a1/src/transformers/models/bert/modeling_bert.py#L456+struct BertOutput {+ dense: Linear,+ layer_norm: LayerNorm,+ dropout: Dropout,+ span: tracing::Span,+}++impl BertOutput {+ fn load(vb: VarBuilder, config: &Config) -> Result<Self> {+ let dense = linear(config.intermediate_size, config.hidden_size, vb.pp("dense"))?;+ let layer_norm = layer_norm(+ config.hidden_size,+ config.layer_norm_eps,+ vb.pp("LayerNorm"),+ )?;+ let dropout = Dropout::new(config.hidden_dropout_prob);+ Ok(Self {+ dense,+ layer_norm,+ dropout,+ span: tracing::span!(tracing::Level::TRACE, "out"),+ })+ }++ fn forward(&self, hidden_states: &Tensor, input_tensor: &Tensor) -> Result<Tensor> {+ let _enter = self.span.enter();+ let hidden_states = self.dense.forward(hidden_states)?;+ let hidden_states = self.dropout.forward(&hidden_states)?;+ self.layer_norm.forward(&(hidden_states + input_tensor)?)+ }+}++// https://github.com/huggingface/transformers/blob/6eedfa6dd15dc1e22a55ae036f681914e5a0d9a1/src/transformers/models/bert/modeling_bert.py#L470+struct BertLayer {+ attention: BertAttention,+ intermediate: BertIntermediate,+ output: BertOutput,+ span: tracing::Span,+}++impl BertLayer {+ fn load(vb: VarBuilder, config: &Config) -> Result<Self> {+ let attention = BertAttention::load(vb.pp("attention"), config)?;+ let intermediate = BertIntermediate::load(vb.pp("intermediate"), config)?;+ let output = BertOutput::load(vb.pp("output"), config)?;+ Ok(Self {+ attention,+ intermediate,+ output,+ span: tracing::span!(tracing::Level::TRACE, "layer"),+ })+ }+}++impl BertLayer {+ fn forward(&self, hidden_states: &Tensor, attention_mask: &Tensor) -> Result<Tensor> {+ let _enter = self.span.enter();+ let attention_output = self.attention.forward(hidden_states, attention_mask)?;+ // TODO: Support cross-attention?+ // https://github.com/huggingface/transformers/blob/6eedfa6dd15dc1e22a55ae036f681914e5a0d9a1/src/transformers/models/bert/modeling_bert.py#L523+ // TODO: Support something similar to `apply_chunking_to_forward`?+ let intermediate_output = self.intermediate.forward(&attention_output)?;+ let layer_output = self+ .output+ .forward(&intermediate_output, &attention_output)?;+ Ok(layer_output)+ }+}++// https://github.com/huggingface/transformers/blob/6eedfa6dd15dc1e22a55ae036f681914e5a0d9a1/src/transformers/models/bert/modeling_bert.py#L556+struct BertEncoder {+ layers: Vec<BertLayer>,+ span: tracing::Span,+}- self.layer_norm.forward(&embeddings)+impl BertEncoder {+ fn load(vb: VarBuilder, config: &Config) -> Result<Self> {+ let layers = (0..config.num_hidden_layers)+ .map(|index| BertLayer::load(vb.pp(&format!("layer.{index}")), config))+ .collect::<Result<Vec<_>>>()?;+ let span = tracing::span!(tracing::Level::TRACE, "encoder");+ Ok(BertEncoder { layers, span })}}-#[cfg(test)]-mod tests {- use super::*;+impl BertEncoder {+ fn forward(&self, hidden_states: &Tensor, attention_mask: &Tensor) -> Result<Tensor> {+ let _enter = self.span.enter();+ let mut hidden_states = hidden_states.clone();+ // Use a loop rather than a fold as it's easier to modify when adding debug/...+ for layer in self.layers.iter() {+ hidden_states = layer.forward(&hidden_states, attention_mask)?+ }+ Ok(hidden_states)+ }+}++// https://github.com/huggingface/transformers/blob/6eedfa6dd15dc1e22a55ae036f681914e5a0d9a1/src/transformers/models/bert/modeling_bert.py#L654+pub struct BertPooler {+ dense: Linear,+}++impl BertPooler {+ fn load(vb: VarBuilder, config: &Config) -> Result<Self> {+ let dense = linear(config.hidden_size, config.hidden_size, vb.pp("dense"))?;+ Ok(Self { dense })+ }+}++impl Module for BertPooler {+ fn forward(&self, hidden_states: &Tensor) -> Result<Tensor> {+ let first_token_tensor = hidden_states.get_on_dim(1, 0)?;+ let pooled_output = self.dense.forward(&first_token_tensor)?;+ pooled_output.tanh()+ }+}++// https://github.com/huggingface/transformers/blob/6eedfa6dd15dc1e22a55ae036f681914e5a0d9a1/src/transformers/models/bert/modeling_bert.py#L874+pub struct BertModel {+ embeddings: BertEmbeddings,+ encoder: BertEncoder,+ pooler: Option<BertPooler>,+ pub device: Device,+ span: tracing::Span,+}++impl BertModel {+ pub fn load(vb: VarBuilder, config: &Config) -> Result<Self> {+ let (embeddings, encoder) = match (+ BertEmbeddings::load(vb.pp("embeddings"), config),+ BertEncoder::load(vb.pp("encoder"), config),+ ) {+ (Ok(embeddings), Ok(encoder)) => (embeddings, encoder),+ (Err(err), _) | (_, Err(err)) => {+ if let Some(model_type) = &config.model_type {+ if let (Ok(embeddings), Ok(encoder)) = (+ BertEmbeddings::load(vb.pp(&format!("{model_type}.embeddings")), config),+ BertEncoder::load(vb.pp(&format!("{model_type}.encoder")), config),+ ) {+ (embeddings, encoder)+ } else {+ return Err(err);+ }+ } else {+ return Err(err);+ }+ }+ };++ let pooler = BertPooler::load(vb.pp("pooler"), config)+ .or_else(|e| {+ if let Some(model_type) = &config.model_type {+ BertPooler::load(vb.pp(&format!("{model_type}.pooler")), config)+ } else {+ Err(e)+ }+ })+ .ok();++ Ok(Self {+ embeddings,+ encoder,+ pooler,+ device: vb.device().clone(),+ span: tracing::span!(tracing::Level::TRACE, "model"),+ })+ }++ pub fn set_pooler(&mut self, pooler: Option<BertPooler>) {+ self.pooler = pooler;+ }++ fn extended_attention_mask(+ &self,+ attention_mask: &Tensor,+ input_ids_dims: &[usize],+ ) -> Result<Tensor> {+ let bs = input_ids_dims[0];+ let seq = input_ids_dims[input_ids_dims.len() - 1];+ let attention_mask = attention_mask.reshape(&[bs, 1, 1, seq])?;++ // TODO: subtract min value of dtype instead of 1e4+ // i.e. something like: (1.0 - attention_mask)? * dtype.min_value()?+ (1.0 - attention_mask)? * -1e4+ }++ pub fn forward(+ &self,+ input_ids: &Tensor,+ token_type_ids: &Tensor,+ attention_mask: &Tensor,+ ) -> Result<Tensor> {+ let _enter = self.span.enter();+ let dims = input_ids.dims();++ if dims.len() <= 1 {+ return Err(candle_core::Error::UnexpectedNumberOfDims {+ expected: 2,+ got: dims.len(),+ shape: input_ids.shape().clone(),+ });+ }++ let attention_mask = self.extended_attention_mask(attention_mask, &dims)?;++ let embedding_output = self.embeddings.forward(input_ids, token_type_ids)?;+ let sequence_output = self.encoder.forward(&embedding_output, &attention_mask)?;++ match &self.pooler {+ Some(pooler) => pooler.forward(&sequence_output),+ None => Ok(sequence_output),+ }+ }}diff --git a/crates/core/src/models/mod.rs b/crates/core/src/models/mod.rsindex f94d32f4..01b31e25 100644--- a/crates/core/src/models/mod.rs+++ b/crates/core/src/models/mod.rs@@ -1 +1 @@-mod bert;+pub mod bert;diff --git a/crates/core/src/ranking/models/cross_encoder.rs b/crates/core/src/ranking/models/cross_encoder.rsindex a434140e..2d6c8e51 100644--- a/crates/core/src/ranking/models/cross_encoder.rs+++ b/crates/core/src/ranking/models/cross_encoder.rs@@ -16,127 +16,134 @@use anyhow::anyhow;use anyhow::Result;+use candle_core::Module;+use candle_core::{Device, Tensor};+use candle_nn::Linear;+use candle_nn::VarBuilder;use std::path::Path;+use tokenizers::PaddingParams;+use tokenizers::TruncationParams;-#[cfg(feature = "libtorch")]-pub use model::*;+use crate::models::bert;+use crate::models::bert::BertModel;-#[cfg(feature = "libtorch")]-mod model {- use super::*;-- use itertools::Itertools;- use tch::Tensor;- use tokenizers::PaddingParams;- use tokenizers::TruncationParams;+const TRUNCATE_INPUT: usize = 128;- const TRUNCATE_INPUT: usize = 128;+pub struct CrossEncoderModel {+ tokenizer: tokenizers::Tokenizer,+ encoder: BertModel,+ classifier: Linear,+ device: Device,+ dtype: candle_core::DType,+}- pub struct CrossEncoderModel {- tokenizer: tokenizers::Tokenizer,- model: tch::CModule,+impl CrossEncoderModel {+ pub fn open<P: AsRef<Path>>(folder: P) -> Result<Self> {+ let device = Device::Cpu;+ let dtype = candle_core::DType::F16;++ let truncation = TruncationParams {+ max_length: TRUNCATE_INPUT,+ ..Default::default()+ };++ let padding = PaddingParams {+ ..Default::default()+ };++ let mut tokenizer =+ tokenizers::Tokenizer::from_file(folder.as_ref().join("tokenizer.json"))+ .map_err(|_| anyhow!("couldn't open tokenizer"))?;++ tokenizer+ .with_truncation(Some(truncation))+ .map_err(|_| anyhow!("tokenizer truncation settings"))?;+ tokenizer.with_padding(Some(padding));++ let vb = unsafe {+ VarBuilder::from_mmaped_safetensors(+ &[folder.as_ref().join("model.safetensors")],+ dtype,+ &device,+ )?+ };+ let config = std::fs::read_to_string(folder.as_ref().join("config.json"))?;+ let mut config: bert::Config = serde_json::from_str(&config)?;+ config.hidden_act = bert::HiddenAct::GeluApproximate;++ let classifier: Linear = candle_nn::linear(config.hidden_size, 1, vb.pp("classifier"))?;++ // all tensors can be loaded with (useful for debugging):+ // candle_core::safetensors::load(folder.as_ref().join("model.safetensors"), &device)++ let encoder = BertModel::load(vb, &config)?;++ Ok(Self {+ tokenizer,+ encoder,+ classifier,+ device,+ dtype,+ })}+}- impl CrossEncoderModel {- pub fn open<P: AsRef<Path>>(folder: P) -> Result<Self> {- let truncation = TruncationParams {- max_length: TRUNCATE_INPUT,- ..Default::default()- };-- let padding = PaddingParams {- ..Default::default()- };-- let mut tokenizer =- tokenizers::Tokenizer::from_file(folder.as_ref().join("tokenizer.json"))- .map_err(|_| anyhow!("couldn't open tokenizer"))?;-- tokenizer- .with_truncation(Some(truncation))- .map_err(|_| anyhow!("tokenizer truncation settings"))?;- tokenizer.with_padding(Some(padding));-- let model = tch::CModule::load(folder.as_ref().join("model.pt"))?;-- Ok(Self { tokenizer, model })+impl CrossEncoder for CrossEncoderModel {+ fn run(&self, query: &str, bodies: &[String]) -> Vec<f64> {+ if bodies.is_empty() {+ return Vec::new();}- }- impl CrossEncoder for CrossEncoderModel {- fn run(&self, query: &str, bodies: &[String]) -> Vec<f64> {- if bodies.is_empty() {- return Vec::new();- }-- let bs = bodies.len();-- let input: Vec<_> = bodies- .iter()- .map(|body| {- (- query.to_string(),- body.split_whitespace()- .take(TRUNCATE_INPUT)- .collect::<String>(),- )- })- .collect();-- let encoded = self.tokenizer.encode_batch(input, true).unwrap();-- let num_tokens = encoded- .iter()- .map(|enc| enc.get_ids().len())- .max()- .unwrap_or(0);-- let ids = encoded- .iter()- .flat_map(|enc| enc.get_ids().iter().map(|i| *i as i64).take(TRUNCATE_INPUT))- .collect_vec();-- let attention_mask = encoded- .iter()- .flat_map(|enc| {- enc.get_attention_mask()- .iter()- .map(|i| *i as i64)- .take(TRUNCATE_INPUT)- })- .collect_vec();-- let type_ids = encoded- .iter()- .flat_map(|enc| {- enc.get_type_ids()- .iter()- .map(|i| *i as i64)+ let input: Vec<_> = bodies+ .iter()+ .map(|body| {+ (+ query.to_string(),+ body.split_whitespace().take(TRUNCATE_INPUT)- })- .collect_vec();-- let ids = Tensor::from_slice(&ids).reshape([bs as i64, num_tokens as i64]);- let attention_mask =- Tensor::from_slice(&attention_mask).reshape([bs as i64, num_tokens as i64]);- let type_ids = Tensor::from_slice(&type_ids).reshape([bs as i64, num_tokens as i64]);-- let output = self- .model- .forward_ts(&[ids, attention_mask, type_ids])- .unwrap();-- let mut res = Vec::with_capacity(bs);-- for i in 0..bs {- let s = output.double_value(&[i as i64, 0]);- let s = s.exp();- let s = s / (s + 1.0);- res.push(s);- }-- res- }+ .collect::<String>(),+ )+ })+ .collect();++ let encoded = self.tokenizer.encode_batch(input, true).unwrap();++ let ids = encoded+ .iter()+ .map(|enc| Tensor::new(enc.get_ids(), &self.device).map_err(|e| anyhow!(e)))+ .collect::<Result<Vec<_>>>()+ .unwrap();+ let input_ids = Tensor::stack(&ids, 0).unwrap();++ let token_type_ids = input_ids.zeros_like().unwrap();++ let attention_mask = encoded+ .iter()+ .map(|enc| Tensor::new(enc.get_attention_mask(), &self.device).map_err(|e| anyhow!(e)))+ .collect::<Result<Vec<_>>>()+ .unwrap();+ let attention_mask = Tensor::stack(&attention_mask, 0)+ .unwrap()+ .to_dtype(self.dtype)+ .unwrap();++ let logits = self+ .encoder+ .forward(&input_ids, &token_type_ids, &attention_mask)+ .unwrap();++ let scores = self+ .classifier+ .forward(&logits)+ .unwrap()+ .squeeze(1)+ .unwrap()+ .to_dtype(candle_core::DType::F64)+ .unwrap();++ let scores = candle_nn::ops::sigmoid(&scores).unwrap();++ scores.to_vec1().unwrap()}}@@ -153,12 +160,15 @@ impl CrossEncoder for DummyCrossEncoder {}#[cfg(test)]-#[cfg(feature = "libtorch")]mod tests {use super::*;#[test]fn sanity_check() {+ if !Path::new("../../data/cross_encoder").exists() {+ return;+ }+let model = CrossEncoderModel::open("../../data/cross_encoder").expect("Failed to find cross-encoder model");@@ -177,14 +187,14 @@ mod tests {);}- assert!(- model.run(- "how many people live in paris",- &["there are currently 1234 people living in paris".to_string()]- )[0] > model.run(- "how many people live in paris",- &["I really like cake and my favorite cake is probably brownie".to_string()]- )[0]+ let res = model.run(+ "how many people live in paris",+ &[+ "there are currently 1234 people living in paris".to_string(),+ "I really like cake and my favorite cake is probably brownie".to_string(),+ ],);++ assert!(res[0] > res[1]);}}diff --git a/crates/core/src/ranking/models/mod.rs b/crates/core/src/ranking/models/mod.rsindex a2b52c47..41e9906e 100644--- a/crates/core/src/ranking/models/mod.rs+++ b/crates/core/src/ranking/models/mod.rs@@ -1,19 +1,3 @@-// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.-pub mod cross_encoder;pub mod lambdamart;pub mod linear;diff --git a/crates/core/src/summarizer.rs b/crates/core/src/summarizer.rsindex 82d28299..901a8c0c 100644--- a/crates/core/src/summarizer.rs+++ b/crates/core/src/summarizer.rs@@ -14,6 +14,9 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+use anyhow::anyhow;+use candle_core::{Device, Tensor};+use candle_nn::VarBuilder;use futures::stream::Stream;use std::{cmp::Reverse,@@ -23,9 +26,12 @@ use std::{};use tokio_stream::StreamExt;-use crate::{llm_utils::OpenAiApi, Result};+use crate::{+ llm_utils::OpenAiApi,+ models::bert::{self, BertModel},+ Result,+};use itertools::{intersperse, Itertools};-use tch::Tensor;use tokenizers::{PaddingParams, TruncationParams};use crate::ceil_char_boundary;@@ -284,12 +290,17 @@ impl Summarizer {}pub struct DualEncoder {- model: tch::CModule,+ model: BertModel,tokenizer: tokenizers::Tokenizer,+ device: Device,+ dtype: candle_core::DType,}impl DualEncoder {pub fn open<P: AsRef<Path>>(folder: P) -> Result<Self> {+ let device = Device::Cpu;+ let dtype = candle_core::DType::F16;+let truncation = TruncationParams {max_length: 256,..Default::default()@@ -305,36 +316,62 @@ impl DualEncoder {tokenizer.with_truncation(Some(truncation)).unwrap();tokenizer.with_padding(Some(padding));- let model = tch::CModule::load(folder.as_ref().join("model.pt"))?;- Ok(Self { model, tokenizer })+ let vb = unsafe {+ VarBuilder::from_mmaped_safetensors(+ &[folder.as_ref().join("model.safetensors")],+ dtype,+ &device,+ )?+ };+ let config = std::fs::read_to_string(folder.as_ref().join("config.json"))?;+ let mut config: bert::Config = serde_json::from_str(&config)?;+ config.hidden_act = bert::HiddenAct::GeluApproximate;++ // all tensors can be loaded with (useful for debugging):+ // candle_core::safetensors::load(folder.as_ref().join("model.safetensors"), &device)++ let mut model = BertModel::load(vb, &config)?;+ model.set_pooler(None); // model should use mean pooling++ Ok(Self {+ model,+ tokenizer,+ device,+ dtype,+ })}- fn embed(&self, text: &str) -> Result<Tensor> {- let query = self.tokenizer.encode(text, false).unwrap();+ pub fn embed(&self, texts: &[&str]) -> Result<Tensor> {+ let enc = self+ .tokenizer+ .encode_batch(texts.to_vec(), true)+ .map_err(|e| anyhow!(e))?;- let ids = query- .get_ids()+ let ids = enc.iter()- .map(|&id| id as i64)- .collect::<Vec<_>>();+ .map(|enc| Tensor::new(enc.get_ids(), &self.device).map_err(|e| anyhow!(e)))+ .collect::<Result<Vec<_>>>()?;- let types = query- .get_type_ids()- .iter()- .map(|&id| id as i64)- .collect::<Vec<_>>();+ let input_ids = Tensor::stack(&ids, 0)?;++ let token_type_ids = input_ids.zeros_like()?;- let mask = query- .get_attention_mask()+ let attention_mask = enc.iter()- .map(|&id| id as i64)- .collect::<Vec<_>>();+ .map(|enc| Tensor::new(enc.get_attention_mask(), &self.device).map_err(|e| anyhow!(e)))+ .collect::<Result<Vec<_>>>()?;+ let attention_mask = Tensor::stack(&attention_mask, 0)?.to_dtype(self.dtype)?;- let ids = Tensor::from_slice(&ids).reshape([1, -1]);- let types = Tensor::from_slice(&types).reshape([1, -1]);- let mask = Tensor::from_slice(&mask).reshape([1, -1]);+ let emb = self+ .model+ .forward(&input_ids, &token_type_ids, &attention_mask)?;- Ok(self.model.forward_ts(&[ids, types, mask])?.squeeze())+ let (_n_sentence, n_tokens, _hidden_size) = emb.dims3()?;++ let emb = (emb.sum(1)? / (n_tokens as f64))?; // mean pooling+ let emb = emb.broadcast_div(&emb.sqr()?.sum_keepdim(1)?.sqrt()?)?; // l2 normalization++ Ok(emb)}}@@ -344,17 +381,25 @@ impl PassageScorer for DualEncoder {type PassageEmbedding = Tensor;fn embed_query(&self, query: &str) -> Option<Self::QueryEmbedding> {- self.embed(query).ok()+ self.embed(&[query]).ok()}fn embed_passage(&self, passage: &str) -> Option<Self::PassageEmbedding> {- self.embed(passage).ok()+ self.embed(&[passage]).ok()}fn score(&self, query: &Self::QueryEmbedding, passage: &Self::PassageEmbedding) -> f32 {- let score = query.dot(&passage.transpose(0, 0));-- score.double_value(&[]) as f32+ query+ .matmul(&passage.t().unwrap())+ .unwrap()+ .get(0)+ .unwrap()+ .squeeze(0)+ .unwrap()+ .to_dtype(candle_core::DType::F32)+ .unwrap()+ .to_vec0()+ .unwrap()}}@@ -497,4 +542,24 @@ mod tests {assert_eq!(p, &text[range]);}}++ #[test]+ fn test_dual_encoder() {+ if !Path::new("../../data/summarizer/dual_encoder").exists() {+ return;+ }++ let model =+ DualEncoder::open("../../data/summarizer/dual_encoder").expect("Failed to load model");+ let query = "What is the capital of France?";+ let pos = "The capital of France is Paris.";+ let neg = "The best baguette in Paris can be found at Boulangerie Pichard.";++ let query_emb = model.embed_query(query).unwrap();+ let pos_emb = model.embed_passage(pos).unwrap();+ let neg_emb = model.embed_passage(neg).unwrap();++ assert!(model.score(&query_emb, &pos_emb) > 0.0);+ assert!(model.score(&query_emb, &pos_emb) > model.score(&query_emb, &neg_emb));+ }}diff --git a/crates/ggml-sys/Cargo.toml b/crates/ggml-sys/Cargo.tomldeleted file mode 100644index fc95fe70..00000000--- a/crates/ggml-sys/Cargo.toml+++ /dev/null@@ -1,11 +0,0 @@-[package]-name = "ggml-sys"-authors = ['Stract <[email protected]>']-version = "0.1.0"-edition = "2021"-license = "MIT"--[build-dependencies]-bindgen = { workspace = true }-cmake = { workspace = true }-cc = { workspace = true }diff --git a/crates/ggml-sys/build.rs b/crates/ggml-sys/build.rsdeleted file mode 100644index 86d6928a..00000000--- a/crates/ggml-sys/build.rs+++ /dev/null@@ -1,45 +0,0 @@-use std::env;-use std::path::PathBuf;--fn generate_bindings() {- let out_path = PathBuf::from(env::var("OUT_DIR").unwrap());- bindgen::builder()- .header("ggml-src/include/ggml/ggml.h")- .header("ggml-src/include/ggml/ggml-alloc.h")- .header("ggml-src/include/ggml/ggml-backend.h")- .parse_callbacks(Box::new(bindgen::CargoCallbacks::new()))- .generate()- .expect("Unable to generate bindings")- .write_to_file(out_path.join("bindings.rs"))- .expect("Couldn't write bindings!");-}--fn main() {- let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap();-- generate_bindings();-- let ggml_src = PathBuf::from("ggml-src")- .join("src")- .canonicalize()- .unwrap();- let mut build = cmake::Config::new(ggml_src);-- build.define("GGML_STATIC", "ON");-- if target_os == "linux" {- build.define("GGML_OPENBLAS", "ON");- println!("cargo:rustc-link-lib=openblas");- }-- if target_os == "macos" {- println!("cargo:rustc-link-lib=framework=Accelerate");- }-- let dst = build.build();-- println!("cargo:rustc-link-search={}/lib", dst.display());-- println!("cargo:rerun-if-changed=ggml-src");- println!("cargo:rustc-link-lib=static=ggml");-}diff --git a/crates/ggml-sys/ggml-src b/crates/ggml-sys/ggml-srcdeleted file mode 160000index ca7e136d..00000000--- a/crates/ggml-sys/ggml-src+++ /dev/null@@ -1 +0,0 @@-Subproject commit ca7e136d7c4cd56399ea399b0757fb3806d91e80diff --git a/crates/ggml-sys/src/lib.rs b/crates/ggml-sys/src/lib.rsdeleted file mode 100644index 514d755c..00000000--- a/crates/ggml-sys/src/lib.rs+++ /dev/null@@ -1,35 +0,0 @@-#![allow(non_upper_case_globals)]-#![allow(non_camel_case_types)]-#![allow(non_snake_case)]--include!(concat!(env!("OUT_DIR"), "/bindings.rs"));--#[cfg(test)]-mod tests {- use super::*;-- #[test]- fn sanity() {- unsafe {- let params = ggml_init_params {- mem_size: 128 * 1024 * 1024,- mem_buffer: std::ptr::null_mut(),- no_alloc: false,- };-- let ctx = ggml_init(params);- let a = ggml_new_f32(ctx, 2.0);- let b = ggml_new_f32(ctx, 2.0);-- let c = ggml_add(ctx, a, b);-- let gf = ggml_new_graph(ctx);- ggml_build_forward_expand(gf, c);-- ggml_graph_compute_with_ctx(ctx, gf, 1);-- let res = ggml_get_f32_1d(c, 0);- assert_eq!(res, 4.0);- }- }-}diff --git a/crates/ggml/Cargo.toml b/crates/ggml/Cargo.tomldeleted file mode 100644index 329ba104..00000000--- a/crates/ggml/Cargo.toml+++ /dev/null@@ -1,11 +0,0 @@-[package]-name = "ggml"-authors = ['Stract <[email protected]>']-version = "0.1.0"-edition = "2021"-license = "MIT"--# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html--[dependencies]-ggml-sys = {path = "../ggml-sys"}diff --git a/crates/ggml/src/context.rs b/crates/ggml/src/context.rsdeleted file mode 100644index a1f83ab2..00000000--- a/crates/ggml/src/context.rs+++ /dev/null@@ -1,125 +0,0 @@-use std::{ptr::NonNull, sync::Arc};--use crate::{Dims, Tensor, ValidDims};--pub(crate) struct InnerContext {- ctx: NonNull<ggml_sys::ggml_context>,-}--impl InnerContext {- fn new(mem_size: usize) -> Self {- let params = ggml_sys::ggml_init_params {- mem_size,- mem_buffer: std::ptr::null_mut(),- no_alloc: false,- };-- let ctx = unsafe { ggml_sys::ggml_init(params) };-- Self {- ctx: NonNull::new(ctx).unwrap(),- }- }-- pub(crate) fn as_ptr(&self) -> *mut ggml_sys::ggml_context {- self.ctx.as_ptr()- }-}--impl Drop for InnerContext {- fn drop(&mut self) {- unsafe {- ggml_sys::ggml_free(self.ctx.as_ptr());- }- }-}--struct InnerGraph {- ptr: NonNull<ggml_sys::ggml_cgraph>,-}--impl InnerGraph {- fn new(ctx: &InnerContext) -> Self {- let ptr = unsafe { ggml_sys::ggml_new_graph(ctx.as_ptr()) };-- Self {- ptr: NonNull::new(ptr).unwrap(),- }- }-- fn as_ptr(&self) -> *mut ggml_sys::ggml_cgraph {- self.ptr.as_ptr()- }-}--pub struct Graph<const OUT_DIMS: usize>-where- Dims<OUT_DIMS>: ValidDims,-{- inner: InnerGraph,- out: Tensor<OUT_DIMS>,-}--impl<const OUT_DIMS: usize> Graph<OUT_DIMS>-where- Dims<OUT_DIMS>: ValidDims,-{- fn inner(&self) -> &InnerGraph {- &self.inner- }-- pub fn build(ctx: &Context, out: Tensor<OUT_DIMS>) -> Self {- let inner = InnerGraph::new(ctx.inner_ctx());-- unsafe {- ggml_sys::ggml_build_forward_expand(inner.as_ptr(), out.as_ptr());- }-- Self { inner, out }- }-- pub fn compute(&mut self, ctx: &Context) {- ctx.compute(self);- }-- pub fn out(&mut self) -> &Tensor<OUT_DIMS> {- &self.out- }-}--pub struct Context {- ctx: Arc<InnerContext>,- n_threads: usize,-}--impl Context {- pub fn new(mem_size: usize, n_threads: usize) -> Self {- let ctx = Arc::new(InnerContext::new(mem_size));-- Self { ctx, n_threads }- }-- pub fn build_graph<const OUT_DIMS: usize>(&self, out: Tensor<OUT_DIMS>) -> Graph<OUT_DIMS>- where- Dims<OUT_DIMS>: ValidDims,- {- Graph::build(self, out)- }-- pub(crate) fn inner_ctx(&self) -> &Arc<InnerContext> {- &self.ctx- }-- pub fn compute<const OUT_DIMS: usize>(&self, graph: &mut Graph<OUT_DIMS>)- where- Dims<OUT_DIMS>: ValidDims,- {- unsafe {- ggml_sys::ggml_graph_compute_with_ctx(- self.ctx.as_ptr(),- graph.inner().as_ptr(),- self.n_threads as i32,- );- }- }-}diff --git a/crates/ggml/src/dims.rs b/crates/ggml/src/dims.rsdeleted file mode 100644index e498527c..00000000--- a/crates/ggml/src/dims.rs+++ /dev/null@@ -1,81 +0,0 @@-pub struct Dims<const DIMS: usize>-where- Self: ValidDims, {}--pub trait ValidDims {}--macro_rules! valid_dims_impl {- ($($dims:literal)*) => {- $(- impl ValidDims for Dims<$dims> {}- )*- };-}--valid_dims_impl!(1 2 3 4);--pub trait DimsGt<const LHS: usize> {}--macro_rules! dims_gt_impl {- ($($lhs:literal, $rhs:literal)*) => {- $(- impl DimsGt<$lhs> for Dims<$rhs>- where- Dims<$rhs>: ValidDims,- {}- )*- };-}--dims_gt_impl!(1, 2);-dims_gt_impl!(1, 3);-dims_gt_impl!(1, 4);-dims_gt_impl!(2, 2);-dims_gt_impl!(2, 3);-dims_gt_impl!(2, 4);-dims_gt_impl!(3, 4);--pub trait DimsLt<const LHS: usize> {}--macro_rules! dims_lt_impl {- ($($lhs:literal, $rhs:literal)*) => {- $(- impl DimsLt<$lhs> for Dims<$rhs>- where- Dims<$rhs>: ValidDims,- {}- )*- };-}--dims_lt_impl!(2, 1);-dims_lt_impl!(3, 1);-dims_lt_impl!(4, 1);-dims_lt_impl!(2, 2);-dims_lt_impl!(3, 2);-dims_lt_impl!(4, 2);-dims_lt_impl!(3, 3);-dims_lt_impl!(4, 3);-dims_lt_impl!(4, 4);--pub trait DimsEq<const LHS: usize> {}--impl<const DIMS: usize> DimsEq<DIMS> for Dims<DIMS> where Dims<DIMS>: ValidDims {}--pub trait DimsPlusOne<const N: usize> {}--macro_rules! dims_plus_one_impl {- ($($n:literal, $res:literal)*) => {- $(- impl DimsPlusOne<$n> for Dims<$res>- where- Dims<$n>: ValidDims,- Dims<$res>: ValidDims,- {}- )*- };-}--dims_plus_one_impl!(1, 2);-dims_plus_one_impl!(2, 3);-dims_plus_one_impl!(3, 4);diff --git a/crates/ggml/src/lib.rs b/crates/ggml/src/lib.rsdeleted file mode 100644index 2c2f7755..00000000--- a/crates/ggml/src/lib.rs+++ /dev/null@@ -1,8 +0,0 @@-mod context;-mod dims;-pub mod nn;-pub mod tensor;--pub use context::*;-pub use dims::*;-pub use tensor::*;diff --git a/crates/ggml/src/nn/attn.rs b/crates/ggml/src/nn/attn.rsdeleted file mode 100644index daf5fc10..00000000--- a/crates/ggml/src/nn/attn.rs+++ /dev/null@@ -1,77 +0,0 @@-use super::Linear;-use crate::{Context, Tensor};--pub struct MultiHeadAttention {- query: Linear,- key: Linear,- value: Linear,- n_embed: u64,- n_heads: u64,- head_size: u64,-}-impl MultiHeadAttention {- pub fn new(ctx: &mut Context, n_embed: u64, n_heads: u64) -> Self {- let head_size = n_embed / n_heads;- let head_emb = n_heads * head_size;-- let query = Linear::new(ctx, n_embed, head_emb);- let key = Linear::new(ctx, n_embed, head_emb);- let value = Linear::new(ctx, n_embed, head_emb);-- Self {- query,- key,- value,- n_embed,- n_heads,- head_size,- }- }-- pub fn forward(&self, input: &Tensor<3>) -> Tensor<3> {- let toks = input.shape()[1];- let batch_size = input.shape()[2];- let head_shape = [self.head_size, self.n_heads, toks, batch_size];-- let query = self- .query- .forward(input)- .reshape(head_shape)- .permute([0, 2, 1, 3]);-- let key = self- .key- .forward(input)- .reshape(head_shape)- .permute([0, 2, 1, 3]);-- let value = self- .value- .forward(input)- .reshape(head_shape)- .permute([0, 2, 1, 3]);-- let kq = (key * query)- .scale(1.0 / (self.n_embed as f32).sqrt())- .softmax();-- let kqv = value.permute([1, 0, 2, 3]).contiguous() * kq;- let out_dims = [self.n_heads * self.head_size, toks, batch_size];-- kqv.contiguous().reshape(out_dims)- }-}-#[cfg(test)]-mod tests {- use super::*;- use crate::GgmlType;-- #[test]- fn test_attention() {- let mut ctx = Context::new(128 * 1024 * 1024, 1);- let input = Tensor::new(&mut ctx, GgmlType::default(), [128, 3, 5]);- let attention = MultiHeadAttention::new(&mut ctx, 128, 12);- let output = attention.forward(&input);- assert_eq!(output.shape(), [120, 3, 5]);- }-}diff --git a/crates/ggml/src/nn/embedding.rs b/crates/ggml/src/nn/embedding.rsdeleted file mode 100644index 5fece377..00000000--- a/crates/ggml/src/nn/embedding.rs+++ /dev/null@@ -1,35 +0,0 @@-use crate::{Context, GgmlType, Tensor};--pub struct Embedding {- tensor: Tensor<2>,-}--impl Embedding {- pub fn new(ctx: &mut Context, vocab_size: u64, embedding_dim: u64) -> Self {- let tensor = Tensor::new(ctx, GgmlType::default(), [embedding_dim, vocab_size]);-- Self { tensor }- }-- pub fn forward(&mut self, input: &Tensor<1>) -> Tensor<2> {- self.tensor.get_rows(input)- }-}--#[cfg(test)]-mod tests {- use super::*;-- #[test]- fn test_embedding() {- let mut ctx = Context::new(128 * 1024 * 1024, 1);-- let mut embedding = Embedding::new(&mut ctx, 10, 128);-- let input = Tensor::new(&mut ctx, GgmlType::I32, [10]);-- let output = embedding.forward(&input);-- assert_eq!(output.shape(), [128, 10]);- }-}diff --git a/crates/ggml/src/nn/layer_norm.rs b/crates/ggml/src/nn/layer_norm.rsdeleted file mode 100644index acc2a81d..00000000--- a/crates/ggml/src/nn/layer_norm.rs+++ /dev/null@@ -1,18 +0,0 @@-use crate::{Dims, Tensor, ValidDims};--pub struct LayerNorm {- eps: f32,-}--impl LayerNorm {- pub fn new(eps: f32) -> Self {- Self { eps }- }-- pub fn forward<const DIMS: usize>(&mut self, input: &Tensor<DIMS>) -> Tensor<DIMS>- where- Dims<DIMS>: ValidDims,- {- input.norm(self.eps)- }-}diff --git a/crates/ggml/src/nn/linear.rs b/crates/ggml/src/nn/linear.rsdeleted file mode 100644index a7cabc61..00000000--- a/crates/ggml/src/nn/linear.rs+++ /dev/null@@ -1,38 +0,0 @@-use crate::{Context, GgmlType, Tensor};--pub struct Linear {- weight: Tensor<3>,- bias: Option<Tensor<2>>,-}--impl Linear {- pub fn new(ctx: &mut Context, input_size: u64, output_size: u64) -> Self {- let weight = Tensor::new(ctx, GgmlType::default(), [input_size, output_size, 1]);-- Self { weight, bias: None }- }-- pub fn forward(&self, input: &Tensor<3>) -> Tensor<3> {- let res = &self.weight * input;-- if let Some(bias) = &self.bias {- &res + &bias.repeat(&res)- } else {- res- }- }-}--#[cfg(test)]-mod tests {- use super::*;-- #[test]- fn test_linear() {- let mut ctx = Context::new(128 * 1024 * 1024, 1);- let input = Tensor::new(&mut ctx, GgmlType::default(), [128, 3, 5]);- let linear = Linear::new(&mut ctx, 128, 256);- let output = linear.forward(&input);- assert_eq!(output.shape(), [256, 3, 5]);- }-}diff --git a/crates/ggml/src/nn/mod.rs b/crates/ggml/src/nn/mod.rsdeleted file mode 100644index 72cb3ee2..00000000--- a/crates/ggml/src/nn/mod.rs+++ /dev/null@@ -1,9 +0,0 @@-mod attn;-mod embedding;-mod layer_norm;-mod linear;--pub use attn::*;-pub use embedding::*;-pub use layer_norm::*;-pub use linear::*;diff --git a/crates/ggml/src/tensor.rs b/crates/ggml/src/tensor.rsdeleted file mode 100644index b30884a6..00000000--- a/crates/ggml/src/tensor.rs+++ /dev/null@@ -1,535 +0,0 @@-use std::ptr::NonNull;-use std::sync::Arc;--use crate::context::InnerContext;-use crate::{Context, Dims, DimsGt, DimsPlusOne, ValidDims};--#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]-#[allow(non_camel_case_types)]-pub enum GgmlType {- #[default]- F32,- F16,- Q4_0,- Q4_1,- Q5_0,- Q5_1,- Q8_0,- Q8_1,- Q2_K,- Q3_K,- Q4_K,- Q5_K,- Q6_K,- Q8_K,- IQ2_XXS,- IQ2_XS,- I8,- I16,- I32,- COUNT,-}--impl GgmlType {- fn as_raw(&self) -> u32 {- match &self {- GgmlType::F32 => 0,- GgmlType::F16 => 1,- GgmlType::Q4_0 => 2,- GgmlType::Q4_1 => 3,- GgmlType::Q5_0 => 6,- GgmlType::Q5_1 => 7,- GgmlType::Q8_0 => 8,- GgmlType::Q8_1 => 9,- GgmlType::Q2_K => 10,- GgmlType::Q3_K => 11,- GgmlType::Q4_K => 12,- GgmlType::Q5_K => 13,- GgmlType::Q6_K => 14,- GgmlType::Q8_K => 15,- GgmlType::IQ2_XXS => 16,- GgmlType::IQ2_XS => 17,- GgmlType::I8 => 18,- GgmlType::I16 => 19,- GgmlType::I32 => 20,- GgmlType::COUNT => 21,- }- }-}--pub struct Tensor<const DIMS: usize>-where- Dims<DIMS>: ValidDims,-{- ctx: Arc<InnerContext>,- type_: GgmlType,- ptr: NonNull<ggml_sys::ggml_tensor>,-}--impl<const DIMS: usize> Tensor<DIMS>-where- Dims<DIMS>: ValidDims,-{- pub fn new(ctx: &mut Context, t: GgmlType, shape: [u64; DIMS]) -> Self {- let ptr = match DIMS {- 1 => unsafe {- ggml_sys::ggml_new_tensor_1d(- ctx.inner_ctx().as_ptr(),- t.as_raw() as u32,- shape[0] as i64,- )- },- 2 => unsafe {- ggml_sys::ggml_new_tensor_2d(- ctx.inner_ctx().as_ptr(),- t.as_raw() as u32,- shape[0] as i64,- shape[1] as i64,- )- },- 3 => unsafe {- ggml_sys::ggml_new_tensor_3d(- ctx.inner_ctx().as_ptr(),- t.as_raw() as u32,- shape[0] as i64,- shape[1] as i64,- shape[2] as i64,- )- },- 4 => unsafe {- ggml_sys::ggml_new_tensor_4d(- ctx.inner_ctx().as_ptr(),- t.as_raw() as u32,- shape[0] as i64,- shape[1] as i64,- shape[2] as i64,- shape[3] as i64,- )- },- _ => unreachable!("{} is not a valid dimension", DIMS),- };-- Self {- ctx: Arc::clone(ctx.inner_ctx()),- type_: t,- ptr: NonNull::new(ptr).unwrap(),- }- }-- pub(crate) fn as_ptr(&self) -> *mut ggml_sys::ggml_tensor {- self.ptr.as_ptr()- }-- pub fn num_bytes(&self) -> usize {- unsafe { ggml_sys::ggml_nbytes(self.ptr.as_ptr()) }- }-- pub fn copy_from_slice(&mut self, slice: &[f32]) {- if slice.len() as u64 != self.num_elements() {- panic!("slice length does not match tensor shape");- }-- if self.type_ != GgmlType::F32 {- panic!("tensor type does not match slice type");- }-- unsafe {- let ptr = self.ptr.as_ptr().as_ref().unwrap().data;- (ptr as *mut f32).copy_from_nonoverlapping(slice.as_ptr(), slice.len());- }- }-- pub fn copy_to_slice(&self, out: &mut [f32]) {- if out.len() as u64 != self.num_elements() {- panic!("slice length does not match tensor shape");- }-- if self.type_ != GgmlType::F32 {- panic!("tensor type does not match slice type");- }-- let data = unsafe {- std::slice::from_raw_parts(- self.ptr.as_ptr().as_ref().unwrap().data as *const f32,- self.num_elements() as usize,- )- };-- out.copy_from_slice(data);- }-- pub fn num_elements(&self) -> u64 {- self.shape().into_iter().product()- }-- pub fn shape(&self) -> [u64; DIMS] {- let shape = unsafe { self.ptr.as_ptr().as_ref().unwrap().ne };-- let mut out = [0; DIMS];-- for i in 0..DIMS {- out[i] = shape[i] as u64;- }-- out- }-- pub fn get_rows<const ROW_DIMS: usize, const OUT_DIMS: usize>(- &self,- rows: &Tensor<ROW_DIMS>,- ) -> Tensor<OUT_DIMS>- where- Dims<ROW_DIMS>: ValidDims,- Dims<OUT_DIMS>: DimsPlusOne<ROW_DIMS> + ValidDims,- {- debug_assert_eq!(rows.type_, GgmlType::I32, "rows tensor must be of type I32");-- let ptr = unsafe {- ggml_sys::ggml_get_rows(self.ctx.as_ptr(), self.ptr.as_ptr(), rows.ptr.as_ptr())- };-- Tensor {- ctx: Arc::clone(&self.ctx),- type_: self.type_,- ptr: NonNull::new(ptr).unwrap(),- }- }-- pub fn repeat<const OUT_DIMS: usize>(&self, rep: &Tensor<OUT_DIMS>) -> Tensor<OUT_DIMS>- where- Dims<OUT_DIMS>: ValidDims,- Dims<OUT_DIMS>: DimsGt<DIMS>,- {- let ptr =- unsafe { ggml_sys::ggml_repeat(self.ctx.as_ptr(), self.ptr.as_ptr(), rep.as_ptr()) };-- Tensor {- ctx: Arc::clone(&self.ctx),- type_: self.type_,- ptr: NonNull::new(ptr).unwrap(),- }- }-- pub fn reshape<const OUT_DIMS: usize>(&self, shape: [u64; OUT_DIMS]) -> Tensor<OUT_DIMS>- where- Dims<OUT_DIMS>: ValidDims,- {- debug_assert_eq!(- shape.into_iter().product::<u64>(),- self.num_elements(),- "reshape must not change the number of elements"- );-- let ptr = match OUT_DIMS {- 1 => unsafe {- ggml_sys::ggml_reshape_1d(self.ctx.as_ptr(), self.ptr.as_ptr(), shape[0] as i64)- },- 2 => unsafe {- ggml_sys::ggml_reshape_2d(- self.ctx.as_ptr(),- self.ptr.as_ptr(),- shape[0] as i64,- shape[1] as i64,- )- },- 3 => unsafe {- ggml_sys::ggml_reshape_3d(- self.ctx.as_ptr(),- self.ptr.as_ptr(),- shape[0] as i64,- shape[1] as i64,- shape[2] as i64,- )- },- 4 => unsafe {- ggml_sys::ggml_reshape_4d(- self.ctx.as_ptr(),- self.ptr.as_ptr(),- shape[0] as i64,- shape[1] as i64,- shape[2] as i64,- shape[3] as i64,- )- },- _ => unreachable!("{} is not a valid dimension", DIMS),- };-- Tensor {- ctx: Arc::clone(&self.ctx),- type_: self.type_,- ptr: NonNull::new(ptr).unwrap(),- }- }-- pub fn permute<const OUT_DIMS: usize>(&self, shape: [u64; OUT_DIMS]) -> Tensor<OUT_DIMS>- where- Dims<OUT_DIMS>: ValidDims,- {- let ptr = match OUT_DIMS {- 1 => unsafe {- ggml_sys::ggml_permute(- self.ctx.as_ptr(),- self.ptr.as_ptr(),- shape[0] as i32,- 1,- 2,- 3,- )- },- 2 => unsafe {- ggml_sys::ggml_permute(- self.ctx.as_ptr(),- self.ptr.as_ptr(),- shape[0] as i32,- shape[1] as i32,- 2,- 3,- )- },- 3 => unsafe {- ggml_sys::ggml_permute(- self.ctx.as_ptr(),- self.ptr.as_ptr(),- shape[0] as i32,- shape[1] as i32,- shape[2] as i32,- 3,- )- },- 4 => unsafe {- ggml_sys::ggml_permute(- self.ctx.as_ptr(),- self.ptr.as_ptr(),- shape[0] as i32,- shape[1] as i32,- shape[2] as i32,- shape[3] as i32,- )- },- _ => unreachable!("{} is not a valid dimension", DIMS),- };-- Tensor {- ctx: Arc::clone(&self.ctx),- type_: self.type_,- ptr: NonNull::new(ptr).unwrap(),- }- }-- pub fn softmax(&self) -> Tensor<DIMS> {- let ptr = unsafe { ggml_sys::ggml_soft_max(self.ctx.as_ptr(), self.ptr.as_ptr()) };-- Tensor {- ctx: Arc::clone(&self.ctx),- type_: self.type_,- ptr: NonNull::new(ptr).unwrap(),- }- }-- pub fn scale(&self, scale: f32) -> Tensor<DIMS> {- let ptr = unsafe { ggml_sys::ggml_scale(self.ctx.as_ptr(), self.ptr.as_ptr(), scale) };-- Tensor {- ctx: Arc::clone(&self.ctx),- type_: self.type_,- ptr: NonNull::new(ptr).unwrap(),- }- }-- pub fn contiguous(&self) -> Tensor<DIMS> {- let ptr = unsafe { ggml_sys::ggml_cont(self.ctx.as_ptr(), self.ptr.as_ptr()) };-- Tensor {- ctx: Arc::clone(&self.ctx),- type_: self.type_,- ptr: NonNull::new(ptr).unwrap(),- }- }-- pub fn as_type(&self, type_: GgmlType) -> Tensor<DIMS> {- let ptr =- unsafe { ggml_sys::ggml_cast(self.ctx.as_ptr(), self.ptr.as_ptr(), type_.as_raw()) };-- Tensor {- ctx: Arc::clone(&self.ctx),- type_,- ptr: NonNull::new(ptr).unwrap(),- }- }-- pub fn norm(&self, eps: f32) -> Tensor<DIMS> {- let ptr = unsafe { ggml_sys::ggml_norm(self.ctx.as_ptr(), self.ptr.as_ptr(), eps) };-- Tensor {- ctx: Arc::clone(&self.ctx),- type_: self.type_,- ptr: NonNull::new(ptr).unwrap(),- }- }-- pub fn load_bytes(&mut self, bytes: &[u8]) {- debug_assert_eq!(- bytes.len(),- self.num_bytes(),- "bytes must be the same size as the tensor"- );-- unsafe {- debug_assert_ne!(- self.as_ptr().as_mut().unwrap().data,- std::ptr::null_mut(),- "tensor must be allocated"- );-- (self.as_ptr().as_mut().unwrap().data as *mut u8)- .copy_from_nonoverlapping(bytes.as_ptr(), self.num_bytes());- }- }-}--macro_rules! std_ops_impl {- ($trait:ident, $std_func:ident, $func:ident) => {- impl<const DIMS: usize> std::ops::$trait for Tensor<DIMS>- where- Dims<DIMS>: ValidDims,- {- type Output = Tensor<DIMS>;- fn $std_func(self, rhs: Self) -> Self::Output {- if self.type_ != rhs.type_ {- panic!("tensor types do not match");- }-- let ptr = unsafe {- ggml_sys::$func(self.ctx.as_ptr(), self.ptr.as_ptr(), rhs.ptr.as_ptr())- };-- Self::Output {- ctx: Arc::clone(&self.ctx),- type_: self.type_,- ptr: NonNull::new(ptr).unwrap(),- }- }- }-- impl<'a, const DIMS: usize> std::ops::$trait<Tensor<DIMS>> for &'a Tensor<DIMS>- where- Dims<DIMS>: ValidDims,- {- type Output = Tensor<DIMS>;- fn $std_func(self, rhs: Tensor<DIMS>) -> Self::Output {- if self.type_ != rhs.type_ {- panic!("tensor types do not match");- }-- let ptr = unsafe {- ggml_sys::$func(self.ctx.as_ptr(), self.ptr.as_ptr(), rhs.ptr.as_ptr())- };-- Self::Output {- ctx: Arc::clone(&self.ctx),- type_: self.type_,- ptr: NonNull::new(ptr).unwrap(),- }- }- }-- impl<'b, const DIMS: usize> std::ops::$trait<&'b Tensor<DIMS>> for Tensor<DIMS>- where- Dims<DIMS>: ValidDims,- {- type Output = Tensor<DIMS>;- fn $std_func(self, rhs: &'b Tensor<DIMS>) -> Self::Output {- if self.type_ != rhs.type_ {- panic!("tensor types do not match");- }-- let ptr = unsafe {- ggml_sys::$func(self.ctx.as_ptr(), self.ptr.as_ptr(), rhs.ptr.as_ptr())- };-- Self::Output {- ctx: Arc::clone(&self.ctx),- type_: self.type_,- ptr: NonNull::new(ptr).unwrap(),- }- }- }-- impl<'a, 'b, const DIMS: usize> std::ops::$trait<&'b Tensor<DIMS>> for &'a Tensor<DIMS>- where- Dims<DIMS>: ValidDims,- {- type Output = Tensor<DIMS>;- fn $std_func(self, rhs: &'b Tensor<DIMS>) -> Self::Output {- if self.type_ != rhs.type_ {- panic!("tensor types do not match");- }-- let ptr = unsafe {- ggml_sys::$func(self.ctx.as_ptr(), self.ptr.as_ptr(), rhs.ptr.as_ptr())- };-- Self::Output {- ctx: Arc::clone(&self.ctx),- type_: self.type_,- ptr: NonNull::new(ptr).unwrap(),- }- }- }- };-}--std_ops_impl!(Add, add, ggml_add);-std_ops_impl!(Sub, sub, ggml_sub);-std_ops_impl!(Mul, mul, ggml_mul_mat);--#[cfg(test)]-mod tests {- use super::*;- use crate::{Context, Graph};- use std::ops::*;-- macro_rules! test_1d {- ($name:ident, $ops:ident, $a:expr, $b:expr, $res:expr) => {- #[test]- fn $name() {- let mut ctx = Context::new(128 * 1024 * 1024, 1);-- let mut at = Tensor::new(&mut ctx, GgmlType::F32, [$a.len() as u64]);- at.copy_from_slice(&$a);-- let mut bt = Tensor::new(&mut ctx, GgmlType::F32, [$b.len() as u64]);- bt.copy_from_slice(&$b);-- let ct = at.$ops(bt);-- let mut graph = Graph::build(&ctx, ct);-- graph.compute(&ctx);-- let mut out = vec![0.0; $res.len()];- graph.out().copy_to_slice(&mut out);-- assert_eq!(out, $res);- }- };- }-- test_1d!(add_1d, add, [1.0, 2.0], [2.0, 2.0], [3.0, 4.0]);- test_1d!(mul_1d, mul, [2.0, 5.0], [2.0, 3.0], [19.0]);- test_1d!(sub_1d, sub, [3.0, 10.0], [2.0, 4.0], [1.0, 6.0]);-- #[test]- fn num_bytes() {- let mut ctx = Context::new(128 * 1024 * 1024, 1);- let at = Tensor::new(&mut ctx, GgmlType::F32, [2, 3, 4]);- assert_eq!(at.num_bytes(), 32 * 2 * 3 * 4 / 8);- }-- #[test]- fn load_bytes() {- let mut ctx = Context::new(128 * 1024 * 1024, 1);- let mut at = Tensor::new(&mut ctx, GgmlType::F32, [2]);- at.load_bytes(&[1, 2, 3, 4, 5, 6, 7, 8]);- }-}diff --git a/scripts/convert_hf_to_gguf.py b/scripts/convert_hf_to_gguf.pydeleted file mode 100755index 63eea815..00000000--- a/scripts/convert_hf_to_gguf.py+++ /dev/null@@ -1,170 +0,0 @@-#!/usr/bin/env python3--from __future__ import annotations--import argparse-import contextlib-import json-import sys-from pathlib import Path-from typing import Any, ContextManager, Iterator, cast--import numpy as np-import torch--import gguf--class Model:- def __init__(self, dir_model: Path, ftype: int, fname_out: Path):- self.dir_model = dir_model- self.ftype = ftype- self.fname_out = fname_out- self.is_safetensors = self._is_model_safetensors()- self.model_name = "model.safetensors" if self.is_safetensors else "pytorch_model.bin"- self.hparams = Model.load_hparams(self.dir_model)- self.gguf_writer = gguf.GGUFWriter(fname_out, gguf.MODEL_ARCH_NAMES[gguf.MODEL_ARCH.BERT], endianess=gguf.GGUFEndian.LITTLE, use_temp_file=False)-- def get_tensors(self) -> Iterator[tuple[str, torch.Tensor]]:- ctx: ContextManager[Any]- if self.is_safetensors:- from safetensors import safe_open- ctx = cast(ContextManager[Any], safe_open(self.dir_model / self.model_name, framework="pt", device="cpu"))- else:- ctx = contextlib.nullcontext(torch.load(str(self.dir_model / self.model_name), map_location="cpu", weights_only=True))-- with ctx as model_part:- for name in model_part.keys():- data = model_part.get_tensor(name) if self.is_safetensors else model_part[name]- yield name, data-- def set_gguf_parameters(self):- self.gguf_writer.add_name(self.dir_model.name)- self.gguf_writer.add_block_count(self.hparams.get(- "n_layers", self.hparams.get("num_hidden_layers", self.hparams.get("n_layer")),- ))- if (n_ctx := self.hparams.get("max_position_embeddings")) is not None:- self.gguf_writer.add_context_length(n_ctx)- if (n_embd := self.hparams.get("hidden_size")) is not None:- self.gguf_writer.add_embedding_length(n_embd)- if (n_ff := self.hparams.get("intermediate_size")) is not None:- self.gguf_writer.add_feed_forward_length(n_ff)- if (n_head := self.hparams.get("num_attention_heads")) is not None:- self.gguf_writer.add_head_count(n_head)- if (n_head_kv := self.hparams.get("num_key_value_heads")) is not None:- self.gguf_writer.add_head_count_kv(n_head_kv)-- if (n_rms_eps := self.hparams.get("rms_norm_eps")) is not None:- self.gguf_writer.add_layer_norm_rms_eps(n_rms_eps)- if (n_experts := self.hparams.get("num_local_experts")) is not None:- self.gguf_writer.add_expert_count(n_experts)- if (n_experts_used := self.hparams.get("num_experts_per_tok")) is not None:- self.gguf_writer.add_expert_used_count(n_experts_used)-- self.gguf_writer.add_parallel_residual(self.hparams.get("use_parallel_residual", True))-- def write_tensors(self):- for name, data_torch in self.get_tensors():- # we don't need these- if name.endswith((".attention.masked_bias", ".attention.bias", ".attention.rotary_emb.inv_freq")):- continue-- old_dtype = data_torch.dtype-- # convert any unsupported data types to float32- if data_torch.dtype not in (torch.float16, torch.float32):- data_torch = data_torch.to(torch.float32)-- data = data_torch.squeeze().numpy()-- n_dims = len(data.shape)- data_dtype = data.dtype-- # if f32 desired, convert any float16 to float32- if self.ftype == 0 and data_dtype == np.float16:- data = data.astype(np.float32)-- # TODO: Why cant we use these float16 as-is? There should be not reason to store float16 as float32- if self.ftype == 1 and data_dtype == np.float16 and n_dims == 1:- data = data.astype(np.float32)-- # if f16 desired, convert any float32 2-dim weight tensors to float16- if self.ftype == 1 and data_dtype == np.float32 and name.endswith(".weight") and n_dims == 2:- data = data.astype(np.float16)-- print(f"{name}, n_dims = {n_dims}, {old_dtype} --> {data.dtype}")-- self.gguf_writer.add_tensor(name, data)-- def write(self):- self.write_tensors()- self.gguf_writer.write_header_to_file()- self.gguf_writer.write_kv_data_to_file()- self.gguf_writer.write_tensors_to_file()- self.gguf_writer.close()-- @staticmethod- def load_hparams(dir_model):- with open(dir_model / "config.json", "r", encoding="utf-8") as f:- return json.load(f)-- def _is_model_safetensors(self) -> bool:- return (self.dir_model / "model.safetensors").exists()---def parse_args() -> argparse.Namespace:- parser = argparse.ArgumentParser(- description="Convert a huggingface model to a GGML compatible file")- parser.add_argument(- "--outtype", type=str, choices=["f32", "f16"], default="f16",- help="output format - use f32 for float32, f16 for float16",- )- parser.add_argument(- "model", type=Path,- help="directory containing model file",- )- parser.add_argument(- "outfile", type=Path,- help="path to write to",- )-- return parser.parse_args()---def main() -> None:- args = parse_args()-- dir_model = args.model-- if not dir_model.is_dir():- print(f'Error: {args.model} is not a directory', file=sys.stderr)- sys.exit(1)-- ftype_map = {- "f32": gguf.GGMLQuantizationType.F32,- "f16": gguf.GGMLQuantizationType.F16,- }-- if args.outfile is not None:- fname_out = args.outfile- else:- # output in the same directory as the model by default- fname_out = dir_model / f'ggml-model-{args.outtype}.gguf'-- print(f"Loading model: {dir_model.name}")-- hparams = Model.load_hparams(dir_model)-- with torch.inference_mode():- model_instance = Model(dir_model, ftype_map[args.outtype], fname_out)-- print("Set model parameters")- model_instance.set_gguf_parameters()-- print(f"Exporting model to '{fname_out}'")- model_instance.write()-- print(f"Model successfully exported to '{fname_out}'")---if __name__ == '__main__':- main()diff --git a/scripts/download_hf.py b/scripts/download_hf.pydeleted file mode 100644index 114ee146..00000000--- a/scripts/download_hf.py+++ /dev/null@@ -1,20 +0,0 @@-from huggingface_hub import snapshot_download-import argparse--parser = argparse.ArgumentParser(description="Download a huggingface model to a local directory")--parser.add_argument(- "local_dir", type=str-)--parser.add_argument(- "repo_id", type=str-)--args = parser.parse_args()--snapshot_download(- repo_id=args.repo_id,- local_dir=args.local_dir,- local_dir_use_symlinks=False,- revision="main")diff --git a/scripts/export_crossencoder b/scripts/export_crossencoderindex d372a82b..efe1fea4 100755--- a/scripts/export_crossencoder+++ b/scripts/export_crossencoder@@ -1,8 +1,7 @@#!.venv/bin/python3import os-import transformers-import torchimport sys+import transformersMODEL = "cross-encoder/ms-marco-TinyBERT-L-2-v2"@@ -12,40 +11,8 @@ if os.path.exists("data/cross_encoder/"):os.system("mkdir data/cross_encoder")-model = transformers.AutoModelForSequenceClassification.from_pretrained(MODEL)--model.eval()-for p in model.parameters():- p.requires_grad_(False)-tokenizer = transformers.AutoTokenizer.from_pretrained(MODEL)-features = tokenizer(- ["How many people live in Berlin?", "How many people live in Berlin?"],- [- "Berlin has a population of 3,520,031 registered inhabitants in an area of 891.82 square kilometers.",- "New York City is famous for the Metropolitan Museum of Art.",- ],- padding="max_length",- truncation=True,- return_tensors="pt",-)-input_ids = features["input_ids"]-attention_mask = features["attention_mask"]-token_type_ids = features["token_type_ids"]-tokenizer.save_pretrained("data/cross_encoder")--def run_model(input_ids, attention_mask, token_type_ids):- res = model(- input_ids=input_ids,- attention_mask=attention_mask,- token_type_ids=token_type_ids,- return_dict=False,- )[0]- return res---traced_model = torch.jit.trace(run_model, ((input_ids, attention_mask, token_type_ids)))--traced_model.save("data/cross_encoder/model.pt")+model.save_pretrained("data/cross_encoder")+tokenizer.save_pretrained("data/cross_encoder")diff --git a/scripts/export_dual_encoder b/scripts/export_dual_encoderindex 393f5407..92e99264 100755--- a/scripts/export_dual_encoder+++ b/scripts/export_dual_encoder@@ -1,21 +1,10 @@#!.venv/bin/python3import os-import transformers-import torch-import torch.nn.functional as Fimport sys+import transformers-# MODEL = "nreimers/MiniLM-L6-H384-uncased"MODEL = "sentence-transformers/all-MiniLM-L6-v2"-question = "Where is amazon rain forest located"-context = """The Amazon rainforest (Portuguese: Floresta Amazônica or Amazônia; Spanish: Selva Amazónica, Amazonía or usually Amazonia; French: Forêt amazonienne; Dutch: Amazoneregenwoud), also known in English as Amazonia or the Amazon Jungle, is a moist broadleaf forest that covers most of the Amazon basin of South America. This basin encompasses 7,000,000 square kilometres (2,700,000 sq mi), of-which 5,500,000 square kilometres (2,100,000 sq mi) are covered by the rainforest. This region includes territory belonging to nine nations.-The majority of the forest is contained within Brazil, with 60% of the rainforest, followed by Peru with 13%, Colombia with 10%, and with-minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana. States or departments in four nations contain-"Amazonas" in their names. The Amazon represents over half of the planet's remaining rainforests, and comprises the largest and most biodiverse tract-of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species."""-if os.path.exists("data/summarizer/dual_encoder/"):print("data/summarizer/dual_encoder/ already exists. Exiting...")sys.exit()@@ -23,48 +12,7 @@ if os.path.exists("data/summarizer/dual_encoder/"):os.system("mkdir data/summarizer/dual_encoder")model = transformers.AutoModel.from_pretrained(MODEL)--model.eval()-for p in model.parameters():- p.requires_grad_(False)-tokenizer = transformers.AutoTokenizer.from_pretrained(MODEL)-tokenizer.save_pretrained("data/summarizer/dual_encoder")--features = tokenizer(- [question, context],- padding="max_length",- truncation=True,- return_tensors="pt",-)-input_ids = features["input_ids"]-attention_mask = features["attention_mask"]-token_type_ids = features["token_type_ids"]--def mean_pooling(model_output, attention_mask):- token_embeddings = model_output[- 0- ] # First element of model_output contains all token embeddings- input_mask_expanded = (- attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()- )- return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(- input_mask_expanded.sum(1), min=1e-9- )---def run_model(input_ids, token_type_ids, attention_mask):- res = model(- input_ids=input_ids,- token_type_ids=token_type_ids,- attention_mask=attention_mask,- return_dict=False,- )- embs = mean_pooling(res, attention_mask)- return F.normalize(embs, p=2, dim=1)---traced_model = torch.jit.trace(run_model, ((input_ids, token_type_ids, attention_mask)))--traced_model.save("data/summarizer/dual_encoder/model.pt")+model.save_pretrained("data/summarizer/dual_encoder")+tokenizer.save_pretrained("data/summarizer/dual_encoder")