The world's first agentic reverse engineer.
impl model framework in rust
Amruth32370ca081a5
diff --git a/Cargo.lock b/Cargo.lockindex f77e72f..f03feae 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2,51 +2,1008 @@# It is not intended for manual editing.version = 4+[[package]]+name = "aho-corasick"+version = "1.1.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"+dependencies = [+ "memchr",+]++[[package]]+name = "arbitrary"+version = "1.4.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1"+dependencies = [+ "derive_arbitrary",+]++[[package]]+name = "autocfg"+version = "1.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"++[[package]]+name = "bit-set"+version = "0.5.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1"+dependencies = [+ "bit-vec",+]++[[package]]+name = "bit-vec"+version = "0.6.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb"++[[package]]+name = "bitflags"+version = "1.3.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"++[[package]]+name = "bitflags"+version = "2.11.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af"++[[package]]+name = "bytemuck"+version = "1.25.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec"+dependencies = [+ "bytemuck_derive",+]++[[package]]+name = "bytemuck_derive"+version = "1.10.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "byteorder"+version = "1.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"++[[package]]+name = "candle-core"+version = "0.8.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "06ccf5ee3532e66868516d9b315f73aec9f34ea1a37ae98514534d458915dbf1"+dependencies = [+ "byteorder",+ "gemm 0.17.1",+ "half",+ "memmap2",+ "num-traits",+ "num_cpus",+ "rand",+ "rand_distr",+ "rayon",+ "safetensors",+ "thiserror",+ "ug",+ "yoke",+ "zip",+]++[[package]]+name = "candle-nn"+version = "0.8.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "be1160c3b63f47d40d91110a3e1e1e566ae38edddbbf492a60b40ffc3bc1ff38"+dependencies = [+ "candle-core",+ "half",+ "num-traits",+ "rayon",+ "safetensors",+ "serde",+ "thiserror",+]++[[package]]+name = "candle-transformers"+version = "0.8.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "94a0900d49f8605e0e7e6693a1f560e6271279de98e5fa369e7abf3aac245020"+dependencies = [+ "byteorder",+ "candle-core",+ "candle-nn",+ "fancy-regex",+ "num-traits",+ "rand",+ "rayon",+ "serde",+ "serde_json",+ "serde_plain",+ "tracing",+]++[[package]]+name = "cfg-if"+version = "1.0.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"++[[package]]+name = "crc32fast"+version = "1.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"+dependencies = [+ "cfg-if",+]++[[package]]+name = "crossbeam-deque"+version = "0.8.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"+dependencies = [+ "crossbeam-epoch",+ "crossbeam-utils",+]++[[package]]+name = "crossbeam-epoch"+version = "0.9.18"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"+dependencies = [+ "crossbeam-utils",+]++[[package]]+name = "crossbeam-utils"+version = "0.8.21"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"++[[package]]+name = "crunchy"+version = "0.2.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"++[[package]]+name = "derive_arbitrary"+version = "1.4.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "dirs"+version = "5.0.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225"+dependencies = [+ "dirs-sys",+]++[[package]]+name = "dirs-sys"+version = "0.4.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c"+dependencies = [+ "libc",+ "option-ext",+ "redox_users",+ "windows-sys 0.48.0",+]++[[package]]+name = "displaydoc"+version = "0.2.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "dyn-stack"+version = "0.10.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "56e53799688f5632f364f8fb387488dd05db9fe45db7011be066fc20e7027f8b"+dependencies = [+ "bytemuck",+ "reborrow",+]++[[package]]+name = "dyn-stack"+version = "0.13.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1c4713e43e2886ba72b8271aa66c93d722116acf7a75555cce11dcde84388fe8"+dependencies = [+ "bytemuck",+ "dyn-stack-macros",+]++[[package]]+name = "dyn-stack-macros"+version = "0.1.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e1d926b4d407d372f141f93bb444696142c29d32962ccbd3531117cf3aa0bfa9"++[[package]]+name = "either"+version = "1.15.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"++[[package]]+name = "enum-as-inner"+version = "0.6.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc"+dependencies = [+ "heck",+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "equivalent"+version = "1.0.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"++[[package]]+name = "fancy-regex"+version = "0.13.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2"+dependencies = [+ "bit-set",+ "regex-automata",+ "regex-syntax",+]++[[package]]+name = "gemm"+version = "0.17.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6ab24cc62135b40090e31a76a9b2766a501979f3070fa27f689c27ec04377d32"+dependencies = [+ "dyn-stack 0.10.0",+ "gemm-c32 0.17.1",+ "gemm-c64 0.17.1",+ "gemm-common 0.17.1",+ "gemm-f16 0.17.1",+ "gemm-f32 0.17.1",+ "gemm-f64 0.17.1",+ "num-complex",+ "num-traits",+ "paste",+ "raw-cpuid 10.7.0",+ "seq-macro",+]++[[package]]+name = "gemm"+version = "0.18.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ab96b703d31950f1aeddded248bc95543c9efc7ac9c4a21fda8703a83ee35451"+dependencies = [+ "dyn-stack 0.13.2",+ "gemm-c32 0.18.2",+ "gemm-c64 0.18.2",+ "gemm-common 0.18.2",+ "gemm-f16 0.18.2",+ "gemm-f32 0.18.2",+ "gemm-f64 0.18.2",+ "num-complex",+ "num-traits",+ "paste",+ "raw-cpuid 11.6.0",+ "seq-macro",+]++[[package]]+name = "gemm-c32"+version = "0.17.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b9c030d0b983d1e34a546b86e08f600c11696fde16199f971cd46c12e67512c0"+dependencies = [+ "dyn-stack 0.10.0",+ "gemm-common 0.17.1",+ "num-complex",+ "num-traits",+ "paste",+ "raw-cpuid 10.7.0",+ "seq-macro",+]++[[package]]+name = "gemm-c32"+version = "0.18.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f6db9fd9f40421d00eea9dd0770045a5603b8d684654816637732463f4073847"+dependencies = [+ "dyn-stack 0.13.2",+ "gemm-common 0.18.2",+ "num-complex",+ "num-traits",+ "paste",+ "raw-cpuid 11.6.0",+ "seq-macro",+]++[[package]]+name = "gemm-c64"+version = "0.17.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fbb5f2e79fefb9693d18e1066a557b4546cd334b226beadc68b11a8f9431852a"+dependencies = [+ "dyn-stack 0.10.0",+ "gemm-common 0.17.1",+ "num-complex",+ "num-traits",+ "paste",+ "raw-cpuid 10.7.0",+ "seq-macro",+]++[[package]]+name = "gemm-c64"+version = "0.18.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dfcad8a3d35a43758330b635d02edad980c1e143dc2f21e6fd25f9e4eada8edf"+dependencies = [+ "dyn-stack 0.13.2",+ "gemm-common 0.18.2",+ "num-complex",+ "num-traits",+ "paste",+ "raw-cpuid 11.6.0",+ "seq-macro",+]++[[package]]+name = "gemm-common"+version = "0.17.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a2e7ea062c987abcd8db95db917b4ffb4ecdfd0668471d8dc54734fdff2354e8"+dependencies = [+ "bytemuck",+ "dyn-stack 0.10.0",+ "half",+ "num-complex",+ "num-traits",+ "once_cell",+ "paste",+ "pulp 0.18.22",+ "raw-cpuid 10.7.0",+ "rayon",+ "seq-macro",+ "sysctl 0.5.5",+]++[[package]]+name = "gemm-common"+version = "0.18.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a352d4a69cbe938b9e2a9cb7a3a63b7e72f9349174a2752a558a8a563510d0f3"+dependencies = [+ "bytemuck",+ "dyn-stack 0.13.2",+ "half",+ "libm",+ "num-complex",+ "num-traits",+ "once_cell",+ "paste",+ "pulp 0.21.5",+ "raw-cpuid 11.6.0",+ "rayon",+ "seq-macro",+ "sysctl 0.6.0",+]++[[package]]+name = "gemm-f16"+version = "0.17.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7ca4c06b9b11952071d317604acb332e924e817bd891bec8dfb494168c7cedd4"+dependencies = [+ "dyn-stack 0.10.0",+ "gemm-common 0.17.1",+ "gemm-f32 0.17.1",+ "half",+ "num-complex",+ "num-traits",+ "paste",+ "raw-cpuid 10.7.0",+ "rayon",+ "seq-macro",+]++[[package]]+name = "gemm-f16"+version = "0.18.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cff95ae3259432f3c3410eaa919033cd03791d81cebd18018393dc147952e109"+dependencies = [+ "dyn-stack 0.13.2",+ "gemm-common 0.18.2",+ "gemm-f32 0.18.2",+ "half",+ "num-complex",+ "num-traits",+ "paste",+ "raw-cpuid 11.6.0",+ "rayon",+ "seq-macro",+]++[[package]]+name = "gemm-f32"+version = "0.17.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e9a69f51aaefbd9cf12d18faf273d3e982d9d711f60775645ed5c8047b4ae113"+dependencies = [+ "dyn-stack 0.10.0",+ "gemm-common 0.17.1",+ "num-complex",+ "num-traits",+ "paste",+ "raw-cpuid 10.7.0",+ "seq-macro",+]++[[package]]+name = "gemm-f32"+version = "0.18.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bc8d3d4385393304f407392f754cd2dc4b315d05063f62cf09f47b58de276864"+dependencies = [+ "dyn-stack 0.13.2",+ "gemm-common 0.18.2",+ "num-complex",+ "num-traits",+ "paste",+ "raw-cpuid 11.6.0",+ "seq-macro",+]++[[package]]+name = "gemm-f64"+version = "0.17.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "aa397a48544fadf0b81ec8741e5c0fba0043008113f71f2034def1935645d2b0"+dependencies = [+ "dyn-stack 0.10.0",+ "gemm-common 0.17.1",+ "num-complex",+ "num-traits",+ "paste",+ "raw-cpuid 10.7.0",+ "seq-macro",+]++[[package]]+name = "gemm-f64"+version = "0.18.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "35b2a4f76ce4b8b16eadc11ccf2e083252d8237c1b589558a49b0183545015bd"+dependencies = [+ "dyn-stack 0.13.2",+ "gemm-common 0.18.2",+ "num-complex",+ "num-traits",+ "paste",+ "raw-cpuid 11.6.0",+ "seq-macro",+]++[[package]]+name = "getrandom"+version = "0.2.17"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"+dependencies = [+ "cfg-if",+ "libc",+ "wasi",+]++[[package]]+name = "getrandom"+version = "0.3.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"+dependencies = [+ "cfg-if",+ "libc",+ "r-efi",+ "wasip2",+]++[[package]]+name = "half"+version = "2.7.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b"+dependencies = [+ "bytemuck",+ "cfg-if",+ "crunchy",+ "num-traits",+ "rand",+ "rand_distr",+ "zerocopy",+]++[[package]]+name = "hashbrown"+version = "0.16.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"++[[package]]+name = "heck"+version = "0.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"++[[package]]+name = "hermit-abi"+version = "0.5.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c"++[[package]]+name = "indexmap"+version = "2.13.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017"+dependencies = [+ "equivalent",+ "hashbrown",+]+[[package]]name = "itoa"version = "1.0.18"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"+checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"++[[package]]+name = "kong-ml"+version = "1.0.0"+dependencies = [+ "candle-core",+ "candle-nn",+ "candle-transformers",+ "dirs",+ "kong-types",+ "safetensors",+ "serde",+ "serde_json",+]++[[package]]+name = "kong-types"+version = "1.0.0"+dependencies = [+ "serde",+ "serde_json",+]++[[package]]+name = "libc"+version = "0.2.183"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d"++[[package]]+name = "libloading"+version = "0.8.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55"+dependencies = [+ "cfg-if",+ "windows-link",+]++[[package]]+name = "libm"+version = "0.2.16"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"++[[package]]+name = "libredox"+version = "0.1.15"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7ddbf48fd451246b1f8c2610bd3b4ac0cc6e149d89832867093ab69a17194f08"+dependencies = [+ "libc",+]++[[package]]+name = "memchr"+version = "2.8.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"++[[package]]+name = "memmap2"+version = "0.9.10"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3"+dependencies = [+ "libc",+ "stable_deref_trait",+]++[[package]]+name = "num"+version = "0.4.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23"+dependencies = [+ "num-bigint",+ "num-complex",+ "num-integer",+ "num-iter",+ "num-rational",+ "num-traits",+]++[[package]]+name = "num-bigint"+version = "0.4.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9"+dependencies = [+ "num-integer",+ "num-traits",+]++[[package]]+name = "num-complex"+version = "0.4.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"+dependencies = [+ "bytemuck",+ "num-traits",+]++[[package]]+name = "num-integer"+version = "0.1.46"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"+dependencies = [+ "num-traits",+]++[[package]]+name = "num-iter"+version = "0.1.45"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf"+dependencies = [+ "autocfg",+ "num-integer",+ "num-traits",+]++[[package]]+name = "num-rational"+version = "0.4.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824"+dependencies = [+ "num-bigint",+ "num-integer",+ "num-traits",+]++[[package]]+name = "num-traits"+version = "0.2.19"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"+dependencies = [+ "autocfg",+ "libm",+]++[[package]]+name = "num_cpus"+version = "1.17.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b"+dependencies = [+ "hermit-abi",+ "libc",+]++[[package]]+name = "num_enum"+version = "0.7.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26"+dependencies = [+ "num_enum_derive",+ "rustversion",+]++[[package]]+name = "num_enum_derive"+version = "0.7.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8"+dependencies = [+ "proc-macro-crate",+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "once_cell"+version = "1.21.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"++[[package]]+name = "option-ext"+version = "0.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"++[[package]]+name = "paste"+version = "1.0.15"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"++[[package]]+name = "pin-project-lite"+version = "0.2.17"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"++[[package]]+name = "ppv-lite86"+version = "0.2.21"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"+dependencies = [+ "zerocopy",+]++[[package]]+name = "proc-macro-crate"+version = "3.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f"+dependencies = [+ "toml_edit",+]++[[package]]+name = "proc-macro2"+version = "1.0.106"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"+dependencies = [+ "unicode-ident",+]++[[package]]+name = "pulp"+version = "0.18.22"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a0a01a0dc67cf4558d279f0c25b0962bd08fc6dec0137699eae304103e882fe6"+dependencies = [+ "bytemuck",+ "libm",+ "num-complex",+ "reborrow",+]++[[package]]+name = "pulp"+version = "0.21.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "96b86df24f0a7ddd5e4b95c94fc9ed8a98f1ca94d3b01bdce2824097e7835907"+dependencies = [+ "bytemuck",+ "cfg-if",+ "libm",+ "num-complex",+ "reborrow",+ "version_check",+]++[[package]]+name = "quote"+version = "1.0.45"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"+dependencies = [+ "proc-macro2",+]++[[package]]+name = "r-efi"+version = "5.3.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"++[[package]]+name = "rand"+version = "0.9.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1"+dependencies = [+ "rand_chacha",+ "rand_core",+]++[[package]]+name = "rand_chacha"+version = "0.9.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"+dependencies = [+ "ppv-lite86",+ "rand_core",+]++[[package]]+name = "rand_core"+version = "0.9.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c"+dependencies = [+ "getrandom 0.3.4",+][[package]]-name = "kong-ml"-version = "1.0.0"+name = "rand_distr"+version = "0.5.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463"dependencies = [- "serde",+ "num-traits",+ "rand",][[package]]-name = "kong-types"-version = "1.0.0"+name = "raw-cpuid"+version = "10.7.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332"dependencies = [- "serde",- "serde_json",+ "bitflags 1.3.2",][[package]]-name = "memchr"-version = "2.8.0"+name = "raw-cpuid"+version = "11.6.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"+checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186"+dependencies = [+ "bitflags 2.11.0",+][[package]]-name = "proc-macro2"-version = "1.0.106"+name = "rayon"+version = "1.11.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"+checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f"dependencies = [- "unicode-ident",+ "either",+ "rayon-core",][[package]]-name = "quote"-version = "1.0.45"+name = "rayon-core"+version = "1.13.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"+checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"dependencies = [- "proc-macro2",+ "crossbeam-deque",+ "crossbeam-utils",+]++[[package]]+name = "reborrow"+version = "0.5.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430"++[[package]]+name = "redox_users"+version = "0.4.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43"+dependencies = [+ "getrandom 0.2.17",+ "libredox",+ "thiserror",+]++[[package]]+name = "regex-automata"+version = "0.4.14"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"+dependencies = [+ "aho-corasick",+ "memchr",+ "regex-syntax",+]++[[package]]+name = "regex-syntax"+version = "0.8.10"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"++[[package]]+name = "rustversion"+version = "1.0.22"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"++[[package]]+name = "safetensors"+version = "0.4.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "44560c11236a6130a46ce36c836a62936dc81ebf8c36a37947423571be0e55b6"+dependencies = [+ "serde",+ "serde_json",+]++[[package]]+name = "same-file"+version = "1.0.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"+dependencies = [+ "winapi-util",]+[[package]]+name = "seq-macro"+version = "0.3.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc"+[[package]]name = "serde"version = "1.0.228"@@ -90,6 +1047,21 @@ dependencies = ["zmij",]+[[package]]+name = "serde_plain"+version = "1.0.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9ce1fc6db65a611022b23a0dec6975d63fb80a302cb3388835ff02c097258d50"+dependencies = [+ "serde",+]++[[package]]+name = "stable_deref_trait"+version = "1.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"+[[package]]name = "syn"version = "2.0.117"@@ -101,12 +1073,369 @@ dependencies = ["unicode-ident",]+[[package]]+name = "synstructure"+version = "0.13.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "sysctl"+version = "0.5.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ec7dddc5f0fee506baf8b9fdb989e242f17e4b11c61dfbb0635b705217199eea"+dependencies = [+ "bitflags 2.11.0",+ "byteorder",+ "enum-as-inner",+ "libc",+ "thiserror",+ "walkdir",+]++[[package]]+name = "sysctl"+version = "0.6.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "01198a2debb237c62b6826ec7081082d951f46dbb64b0e8c7649a452230d1dfc"+dependencies = [+ "bitflags 2.11.0",+ "byteorder",+ "enum-as-inner",+ "libc",+ "thiserror",+ "walkdir",+]++[[package]]+name = "thiserror"+version = "1.0.69"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"+dependencies = [+ "thiserror-impl",+]++[[package]]+name = "thiserror-impl"+version = "1.0.69"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "toml_datetime"+version = "1.1.0+spec-1.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "97251a7c317e03ad83774a8752a7e81fb6067740609f75ea2b585b569a59198f"+dependencies = [+ "serde_core",+]++[[package]]+name = "toml_edit"+version = "0.25.8+spec-1.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "16bff38f1d86c47f9ff0647e6838d7bb362522bdf44006c7068c2b1e606f1f3c"+dependencies = [+ "indexmap",+ "toml_datetime",+ "toml_parser",+ "winnow",+]++[[package]]+name = "toml_parser"+version = "1.1.0+spec-1.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2334f11ee363607eb04df9b8fc8a13ca1715a72ba8662a26ac285c98aabb4011"+dependencies = [+ "winnow",+]++[[package]]+name = "tracing"+version = "0.1.44"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100"+dependencies = [+ "pin-project-lite",+ "tracing-attributes",+ "tracing-core",+]++[[package]]+name = "tracing-attributes"+version = "0.1.31"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "tracing-core"+version = "0.1.36"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a"+dependencies = [+ "once_cell",+]++[[package]]+name = "ug"+version = "0.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "03719c61a91b51541f076dfdba45caacf750b230cefaa4b32d6f5411c3f7f437"+dependencies = [+ "gemm 0.18.2",+ "half",+ "libloading",+ "memmap2",+ "num",+ "num-traits",+ "num_cpus",+ "rayon",+ "safetensors",+ "serde",+ "thiserror",+ "tracing",+ "yoke",+]+[[package]]name = "unicode-ident"version = "1.0.24"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"+[[package]]+name = "version_check"+version = "0.9.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"++[[package]]+name = "walkdir"+version = "2.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"+dependencies = [+ "same-file",+ "winapi-util",+]++[[package]]+name = "wasi"+version = "0.11.1+wasi-snapshot-preview1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"++[[package]]+name = "wasip2"+version = "1.0.2+wasi-0.2.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5"+dependencies = [+ "wit-bindgen",+]++[[package]]+name = "winapi-util"+version = "0.1.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"+dependencies = [+ "windows-sys 0.61.2",+]++[[package]]+name = "windows-link"+version = "0.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"++[[package]]+name = "windows-sys"+version = "0.48.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"+dependencies = [+ "windows-targets",+]++[[package]]+name = "windows-sys"+version = "0.61.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"+dependencies = [+ "windows-link",+]++[[package]]+name = "windows-targets"+version = "0.48.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"+dependencies = [+ "windows_aarch64_gnullvm",+ "windows_aarch64_msvc",+ "windows_i686_gnu",+ "windows_i686_msvc",+ "windows_x86_64_gnu",+ "windows_x86_64_gnullvm",+ "windows_x86_64_msvc",+]++[[package]]+name = "windows_aarch64_gnullvm"+version = "0.48.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"++[[package]]+name = "windows_aarch64_msvc"+version = "0.48.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"++[[package]]+name = "windows_i686_gnu"+version = "0.48.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"++[[package]]+name = "windows_i686_msvc"+version = "0.48.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"++[[package]]+name = "windows_x86_64_gnu"+version = "0.48.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"++[[package]]+name = "windows_x86_64_gnullvm"+version = "0.48.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"++[[package]]+name = "windows_x86_64_msvc"+version = "0.48.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"++[[package]]+name = "winnow"+version = "1.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a90e88e4667264a994d34e6d1ab2d26d398dcdca8b7f52bec8668957517fc7d8"+dependencies = [+ "memchr",+]++[[package]]+name = "wit-bindgen"+version = "0.51.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5"++[[package]]+name = "yoke"+version = "0.7.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40"+dependencies = [+ "serde",+ "stable_deref_trait",+ "yoke-derive",+ "zerofrom",+]++[[package]]+name = "yoke-derive"+version = "0.7.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+ "synstructure",+]++[[package]]+name = "zerocopy"+version = "0.8.47"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "efbb2a062be311f2ba113ce66f697a4dc589f85e78a4aea276200804cea0ed87"+dependencies = [+ "zerocopy-derive",+]++[[package]]+name = "zerocopy-derive"+version = "0.8.47"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0e8bc7269b54418e7aeeef514aa68f8690b8c0489a06b0136e5f57c4c5ccab89"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "zerofrom"+version = "0.1.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5"+dependencies = [+ "zerofrom-derive",+]++[[package]]+name = "zerofrom-derive"+version = "0.1.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+ "synstructure",+]++[[package]]+name = "zip"+version = "1.1.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9cc23c04387f4da0374be4533ad1208cbb091d5c11d070dfef13676ad6497164"+dependencies = [+ "arbitrary",+ "crc32fast",+ "crossbeam-utils",+ "displaydoc",+ "indexmap",+ "num_enum",+ "thiserror",+]+[[package]]name = "zmij"version = "1.0.21"diff --git a/crates/kong-ml/Cargo.toml b/crates/kong-ml/Cargo.tomlindex 05b24bc..602155b 100644--- a/crates/kong-ml/Cargo.toml+++ b/crates/kong-ml/Cargo.toml@@ -2,7 +2,13 @@name = "kong-ml"version.workspace = trueedition.workspace = true-license.workspace = true[dependencies]-serde.workspace = true\ No newline at end of file+kong-types = { path = "../kong-types" }+candle-core = "0.8"+candle-nn = "0.8"+candle-transformers = "0.8"+safetensors = "0.4"+serde.workspace = true+serde_json.workspace = true+dirs = "5"\ No newline at end of filediff --git a/crates/kong-ml/src/config.rs b/crates/kong-ml/src/config.rsnew file mode 100644index 0000000..321090c--- /dev/null+++ b/crates/kong-ml/src/config.rs@@ -0,0 +1,53 @@+use serde::{Deserialize, Serialize};+use std::path::Path;+use std::error::Error;+use std::fs::read_to_string;+use serde_json;+use candle_transformers::models::bert::Config as BertConfig;+#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct XdaConfig {+ pub vocab_size: usize,+ pub hidden_size: usize,+ pub num_hidden_layers: usize,+ pub num_attention_heads: usize,+ pub intermediate_size: usize,+ pub max_position_embeddings: usize,+ pub num_labels: usize,+ pub classifier_hidden: usize,+}++impl Default for XdaConfig {+ fn default() -> Self {+ Self {+ vocab_size: 261,+ hidden_size: 768,+ num_hidden_layers: 12,+ num_attention_heads: 12,+ intermediate_size: 3072,+ max_position_embeddings: 514,+ num_labels: 3,+ classifier_hidden: 256,+ }+ }+}++impl XdaConfig {+ pub fn from_json(path: &Path) -> Result<Self, Box<dyn Error>> {+ let content = read_to_string(path)?;+ let config: Self = serde_json::from_str(&content)?;+ Ok(config)+ }++ pub fn to_bert_config(&self) -> BertConfig {+ BertConfig {+ vocab_size: self.vocab_size,+ hidden_size: self.hidden_size,+ num_hidden_layers: self.num_hidden_layers,+ num_attention_heads: self.num_attention_heads,+ intermediate_size: self.intermediate_size,+ max_position_embeddings: self.max_position_embeddings,+ hidden_dropout_prob: 0.0,+ ..Default::default()+ }+ }+}\ No newline at end of filediff --git a/crates/kong-ml/src/lib.rs b/crates/kong-ml/src/lib.rsnew file mode 100644index 0000000..c07dc45--- /dev/null+++ b/crates/kong-ml/src/lib.rs@@ -0,0 +1,2 @@+pub mod tokenizer;+pub mod config;\ No newline at end of filediff --git a/crates/kong-ml/src/tokenizer.rs b/crates/kong-ml/src/tokenizer.rsnew file mode 100644index 0000000..1fd40ce--- /dev/null+++ b/crates/kong-ml/src/tokenizer.rs@@ -0,0 +1,89 @@+pub struct ByteTokenizer {+ pub pad_id: u32,+ pub cls_id: u32,+ pub sep_id: u32,+ pub mask_id: u32,+ pub unk_id: u32,+ pub vocab_size: u32,+}++impl Default for ByteTokenizer {+ fn default() -> Self {+ Self {+ pad_id: 256,+ cls_id: 257,+ sep_id: 258,+ mask_id: 259,+ unk_id: 260,+ vocab_size: 261,+ }+ }+}++impl ByteTokenizer {+ pub fn new() -> Self {+ Self::default()+ }++ /// If `max_length` is less than the length of the encoded bytes, truncate the encoded bytes to `max_length`.+ /// Otherwise, pad the encoded bytes to `max_length` with the pad token.+ pub fn encode(&self, bytes: &[u8], max_length: Option<usize>) -> Vec<u32> {+ let mut ids = Vec::with_capacity(bytes.len() + 2);+ ids.push(self.cls_id);+ ids.extend(bytes.iter().map(|&b| b as u32));+ ids.push(self.sep_id);++ if let Some(max_len) = max_length {+ ids.truncate(max_len);+ ids.resize(max_len, self.pad_id);+ }++ ids+ }++ pub fn attention_mask(&self, token_ids: &[u32]) -> Vec<u32> {+ token_ids+ .iter()+ .map(|&id| if id == self.pad_id { 0 } else { 1 })+ .collect()+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn test_special_token_ids() {+ let tok = ByteTokenizer::new();+ assert_eq!(tok.pad_id, 256);+ assert_eq!(tok.cls_id, 257);+ assert_eq!(tok.sep_id, 258);+ assert_eq!(tok.vocab_size, 261);+ }++ #[test]+ fn test_encode_bytes() {+ let tok = ByteTokenizer::new();+ let raw = [0x55u8, 0x48, 0x89, 0xe5];+ let ids = tok.encode(&raw, None);+ assert_eq!(ids, vec![257, 0x55, 0x48, 0x89, 0xe5, 258]);+ }++ #[test]+ fn test_encode_with_padding() {+ let tok = ByteTokenizer::new();+ let raw = [0x55u8, 0x48];+ let ids = tok.encode(&raw, Some(8));+ assert_eq!(ids, vec![257, 0x55, 0x48, 258, 256, 256, 256, 256]);+ }++ #[test]+ fn test_attention_mask() {+ let tok = ByteTokenizer::new();+ let raw = [0x55u8, 0x48];+ let ids = tok.encode(&raw, Some(8));+ let mask = tok.attention_mask(&ids);+ assert_eq!(mask, vec![1, 1, 1, 1, 0, 0, 0, 0]);+ }+}\ No newline at end of file