web search done right
implement some layers in ggml linear, embedding and multihead attention
Mikkel Denkerea3b7a409927
diff --git a/.vscode/settings.json b/.vscode/settings.jsonindex 30cd3a46..d10c3861 100644--- a/.vscode/settings.json+++ b/.vscode/settings.json@@ -15,14 +15,12 @@"editor.defaultFormatter": "esbenp.prettier-vscode"},"prettier.configPath": "frontend/.prettierrc",- "prettier.documentSelectors": [- "**/*.svelte"- ],+ "prettier.documentSelectors": ["**/*.svelte"],"svelte.enable-ts-plugin": true,"tailwindCSS.experimental.configFile": "frontend/tailwind.config.ts","rust-analyzer.linkedProjects": ["Cargo.toml","optics-lsp/Cargo.toml",- "./core/Cargo.toml"+ "./crates/core/Cargo.toml"]}diff --git a/crates/ggml/src/context.rs b/crates/ggml/src/context.rsindex a65b62d1..18f5b53c 100644--- a/crates/ggml/src/context.rs+++ b/crates/ggml/src/context.rs@@ -52,7 +52,7 @@ impl InnerGraph {pub struct Graph<const OUT_DIMS: usize>where- Dims<OUT_DIMS>: ValidDims<OUT_DIMS>,+ Dims<OUT_DIMS>: ValidDims,{inner: InnerGraph,out: Tensor<OUT_DIMS>,@@ -60,7 +60,7 @@ whereimpl<const OUT_DIMS: usize> Graph<OUT_DIMS>where- Dims<OUT_DIMS>: ValidDims<OUT_DIMS>,+ Dims<OUT_DIMS>: ValidDims,{fn inner(&self) -> &InnerGraph {&self.inner@@ -99,7 +99,7 @@ impl Context {pub fn build_graph<const OUT_DIMS: usize>(&self, out: Tensor<OUT_DIMS>) -> Graph<OUT_DIMS>where- Dims<OUT_DIMS>: ValidDims<OUT_DIMS>,+ Dims<OUT_DIMS>: ValidDims,{Graph::build(self, out)}@@ -110,7 +110,7 @@ impl Context {pub fn compute<const OUT_DIMS: usize>(&self, graph: &mut Graph<OUT_DIMS>)where- Dims<OUT_DIMS>: ValidDims<OUT_DIMS>,+ Dims<OUT_DIMS>: ValidDims,{unsafe {ggml_sys::ggml_graph_compute_with_ctx(diff --git a/crates/ggml/src/dims.rs b/crates/ggml/src/dims.rsindex 7e42b049..e498527c 100644--- a/crates/ggml/src/dims.rs+++ b/crates/ggml/src/dims.rs@@ -1,27 +1,27 @@pub struct Dims<const DIMS: usize>where- Self: ValidDims<DIMS>, {}+ Self: ValidDims, {}-pub trait ValidDims<const DIMS: usize> {}+pub trait ValidDims {}macro_rules! valid_dims_impl {($($dims:literal)*) => {$(- impl ValidDims<$dims> for Dims<$dims> {}+ impl ValidDims for Dims<$dims> {})*};}valid_dims_impl!(1 2 3 4);-trait DimsGt<const LHS: usize> {}+pub trait DimsGt<const LHS: usize> {}macro_rules! dims_gt_impl {($($lhs:literal, $rhs:literal)*) => {$(impl DimsGt<$lhs> for Dims<$rhs>where- Dims<$rhs>: ValidDims<$rhs>,+ Dims<$rhs>: ValidDims,{})*};@@ -35,14 +35,14 @@ dims_gt_impl!(2, 3);dims_gt_impl!(2, 4);dims_gt_impl!(3, 4);-trait DimsLt<const LHS: usize> {}+pub trait DimsLt<const LHS: usize> {}macro_rules! dims_lt_impl {($($lhs:literal, $rhs:literal)*) => {$(impl DimsLt<$lhs> for Dims<$rhs>where- Dims<$rhs>: ValidDims<$rhs>,+ Dims<$rhs>: ValidDims,{})*};@@ -58,6 +58,24 @@ dims_lt_impl!(3, 3);dims_lt_impl!(4, 3);dims_lt_impl!(4, 4);-trait DimsEq<const LHS: usize> {}+pub trait DimsEq<const LHS: usize> {}-impl<const DIMS: usize> DimsEq<DIMS> for Dims<DIMS> where Dims<DIMS>: ValidDims<DIMS> {}+impl<const DIMS: usize> DimsEq<DIMS> for Dims<DIMS> where Dims<DIMS>: ValidDims {}++pub trait DimsPlusOne<const N: usize> {}++macro_rules! dims_plus_one_impl {+ ($($n:literal, $res:literal)*) => {+ $(+ impl DimsPlusOne<$n> for Dims<$res>+ where+ Dims<$n>: ValidDims,+ Dims<$res>: ValidDims,+ {}+ )*+ };+}++dims_plus_one_impl!(1, 2);+dims_plus_one_impl!(2, 3);+dims_plus_one_impl!(3, 4);diff --git a/crates/ggml/src/lib.rs b/crates/ggml/src/lib.rsindex 54a50c5f..2c2f7755 100644--- a/crates/ggml/src/lib.rs+++ b/crates/ggml/src/lib.rs@@ -1,6 +1,7 @@mod context;mod dims;-mod tensor;+pub mod nn;+pub mod tensor;pub use context::*;pub use dims::*;diff --git a/crates/ggml/src/nn/attn.rs b/crates/ggml/src/nn/attn.rsnew file mode 100644index 00000000..baf15b16--- /dev/null+++ b/crates/ggml/src/nn/attn.rs@@ -0,0 +1,75 @@+use super::Linear;+use crate::{Context, GgmlType, Tensor};++pub struct MultiheadAttention {+ query: Linear,+ key: Linear,+ value: Linear,+ n_embed: u64,+ n_heads: u64,+ head_size: u64,+}+impl MultiheadAttention {+ pub fn new(ctx: &mut Context, t: GgmlType, n_embed: u64, n_heads: u64) -> Self {+ let head_size = n_embed / n_heads;+ let head_emb = n_heads * head_size;++ let query = Linear::new(ctx, t, n_embed, head_emb);+ let key = Linear::new(ctx, t, n_embed, head_emb);+ let value = Linear::new(ctx, t, n_embed, head_emb);++ Self {+ query,+ key,+ value,+ n_embed,+ n_heads,+ head_size,+ }+ }++ pub fn forward(&self, input: &Tensor<3>) -> Tensor<3> {+ let toks = input.shape()[1];+ let batch_size = input.shape()[2];+ let head_shape = [self.head_size, self.n_heads, toks, batch_size];++ let query = self+ .query+ .forward(input)+ .reshape(head_shape)+ .permute([0, 2, 1, 3]);++ let key = self+ .key+ .forward(input)+ .reshape(head_shape)+ .permute([0, 2, 1, 3]);++ let value = self+ .value+ .forward(input)+ .reshape(head_shape)+ .permute([0, 2, 1, 3]);++ let kq = (key * query)+ .scale(1.0 / (self.n_embed as f32).sqrt())+ .softmax();++ let kqv = value.permute([1, 0, 2, 3]).contiguous() * kq;+ let out_dims = [self.n_heads * self.head_size, toks, batch_size];++ kqv.contiguous().reshape(out_dims)+ }+}+#[cfg(test)]+mod tests {+ use super::*;+ #[test]+ fn test_attention() {+ let mut ctx = Context::new(128 * 1024 * 1024, 1);+ let input = Tensor::new(&mut ctx, GgmlType::F32, [128, 3, 5]);+ let attention = MultiheadAttention::new(&mut ctx, GgmlType::F32, 128, 12);+ let output = attention.forward(&input);+ assert_eq!(output.shape(), [120, 3, 5]);+ }+}diff --git a/crates/ggml/src/nn/embedding.rs b/crates/ggml/src/nn/embedding.rsnew file mode 100644index 00000000..4629b8b0--- /dev/null+++ b/crates/ggml/src/nn/embedding.rs@@ -0,0 +1,35 @@+use crate::{Context, GgmlType, Tensor};++pub struct Embedding {+ tensor: Tensor<2>,+}++impl Embedding {+ pub fn new(ctx: &mut Context, t: GgmlType, vocab_size: u64, embedding_dim: u64) -> Self {+ let tensor = Tensor::new(ctx, t, [embedding_dim, vocab_size]);++ Self { tensor }+ }++ pub fn forward(&mut self, input: &Tensor<1>) -> Tensor<2> {+ self.tensor.get_rows(input)+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn test_embedding() {+ let mut ctx = Context::new(128 * 1024 * 1024, 1);++ let mut embedding = Embedding::new(&mut ctx, GgmlType::F32, 10, 128);++ let input = Tensor::new(&mut ctx, GgmlType::I32, [10]);++ let output = embedding.forward(&input);++ assert_eq!(output.shape(), [128, 10]);+ }+}diff --git a/crates/ggml/src/nn/linear.rs b/crates/ggml/src/nn/linear.rsnew file mode 100644index 00000000..f3dbdde7--- /dev/null+++ b/crates/ggml/src/nn/linear.rs@@ -0,0 +1,38 @@+use crate::{Context, GgmlType, Tensor};++pub struct Linear {+ weight: Tensor<3>,+ bias: Option<Tensor<2>>,+}++impl Linear {+ pub fn new(ctx: &mut Context, t: GgmlType, input_size: u64, output_size: u64) -> Self {+ let weight = Tensor::new(ctx, t, [input_size, output_size, 1]);++ Self { weight, bias: None }+ }++ pub fn forward(&self, input: &Tensor<3>) -> Tensor<3> {+ let res = &self.weight * input;++ if let Some(bias) = &self.bias {+ &res + &bias.repeat(&res)+ } else {+ res+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn test_linear() {+ let mut ctx = Context::new(128 * 1024 * 1024, 1);+ let input = Tensor::new(&mut ctx, GgmlType::F32, [128, 3, 5]);+ let linear = Linear::new(&mut ctx, GgmlType::F32, 128, 256);+ let output = linear.forward(&input);+ assert_eq!(output.shape(), [256, 3, 5]);+ }+}diff --git a/crates/ggml/src/nn/mod.rs b/crates/ggml/src/nn/mod.rsnew file mode 100644index 00000000..009d40d6--- /dev/null+++ b/crates/ggml/src/nn/mod.rs@@ -0,0 +1,7 @@+mod attn;+mod embedding;+mod linear;++pub use attn::*;+pub use embedding::*;+pub use linear::*;diff --git a/crates/ggml/src/tensor.rs b/crates/ggml/src/tensor.rsindex e5c22f4f..80797d63 100644--- a/crates/ggml/src/tensor.rs+++ b/crates/ggml/src/tensor.rs@@ -2,7 +2,7 @@ use std::ptr::NonNull;use std::sync::Arc;use crate::context::InnerContext;-use crate::{Context, Dims, ValidDims};+use crate::{Context, Dims, DimsGt, DimsPlusOne, ValidDims};#[derive(Debug, Clone, Copy, PartialEq, Eq)]#[allow(non_camel_case_types)]@@ -36,39 +36,38 @@ impl GgmlType {GgmlType::F16 => 1,GgmlType::Q4_0 => 2,GgmlType::Q4_1 => 3,- GgmlType::Q5_0 => todo!(),- GgmlType::Q5_1 => todo!(),- GgmlType::Q8_0 => todo!(),- GgmlType::Q8_1 => todo!(),- GgmlType::Q2_K => todo!(),- GgmlType::Q3_K => todo!(),- GgmlType::Q4_K => todo!(),- GgmlType::Q5_K => todo!(),- GgmlType::Q6_K => todo!(),- GgmlType::Q8_K => todo!(),- GgmlType::IQ2_XXS => todo!(),- GgmlType::IQ2_XS => todo!(),- GgmlType::I8 => todo!(),- GgmlType::I16 => todo!(),- GgmlType::I32 => todo!(),- GgmlType::COUNT => todo!(),+ GgmlType::Q5_0 => 6,+ GgmlType::Q5_1 => 7,+ GgmlType::Q8_0 => 8,+ GgmlType::Q8_1 => 9,+ GgmlType::Q2_K => 10,+ GgmlType::Q3_K => 11,+ GgmlType::Q4_K => 12,+ GgmlType::Q5_K => 13,+ GgmlType::Q6_K => 14,+ GgmlType::Q8_K => 15,+ GgmlType::IQ2_XXS => 16,+ GgmlType::IQ2_XS => 17,+ GgmlType::I8 => 18,+ GgmlType::I16 => 19,+ GgmlType::I32 => 20,+ GgmlType::COUNT => 21,}}}pub struct Tensor<const DIMS: usize>where- Dims<DIMS>: ValidDims<DIMS>,+ Dims<DIMS>: ValidDims,{ctx: Arc<InnerContext>,type_: GgmlType,ptr: NonNull<ggml_sys::ggml_tensor>,- shape: [u64; DIMS],}impl Tensor<1> {pub fn copy_from_slice(&mut self, slice: &[f32]) {- if slice.len() != self.shape[0] as usize {+ if slice.len() != self.shape()[0] as usize {panic!("slice length does not match tensor shape");}@@ -83,7 +82,7 @@ impl Tensor<1> {}pub fn copy_to_slice(&self, out: &mut [f32]) {- if out.len() != self.shape[0] as usize {+ if out.len() != self.shape()[0] as usize {panic!("slice length does not match tensor shape");}@@ -94,7 +93,7 @@ impl Tensor<1> {let data = unsafe {std::slice::from_raw_parts(self.ptr.as_ptr().as_ref().unwrap().data as *const f32,- self.shape[0] as usize,+ self.shape()[0] as usize,)};@@ -104,7 +103,7 @@ impl Tensor<1> {impl<const DIMS: usize> Tensor<DIMS>where- Dims<DIMS>: ValidDims<DIMS>,+ Dims<DIMS>: ValidDims,{pub fn new(ctx: &mut Context, t: GgmlType, shape: [u64; DIMS]) -> Self {let ptr = match DIMS {@@ -148,7 +147,6 @@ whereSelf {ctx: Arc::clone(ctx.inner_ctx()),type_: t,- shape,ptr: NonNull::new(ptr).unwrap(),}}@@ -156,31 +154,233 @@ wherepub(crate) fn as_ptr(&self) -> *mut ggml_sys::ggml_tensor {self.ptr.as_ptr()}++ pub fn shape(&self) -> [u64; DIMS] {+ let shape = unsafe { self.ptr.as_ptr().as_ref().unwrap().ne };++ let mut out = [0; DIMS];++ for i in 0..DIMS {+ out[i] = shape[i] as u64;+ }++ out+ }++ pub fn get_rows<const ROW_DIMS: usize, const OUT_DIMS: usize>(+ &self,+ rows: &Tensor<ROW_DIMS>,+ ) -> Tensor<OUT_DIMS>+ where+ Dims<ROW_DIMS>: ValidDims,+ Dims<OUT_DIMS>: DimsPlusOne<ROW_DIMS> + ValidDims,+ {+ assert_eq!(rows.type_, GgmlType::I32, "rows tensor must be of type I32");++ let ptr = unsafe {+ ggml_sys::ggml_get_rows(self.ctx.as_ptr(), self.ptr.as_ptr(), rows.ptr.as_ptr())+ };++ Tensor {+ ctx: Arc::clone(&self.ctx),+ type_: self.type_,+ ptr: NonNull::new(ptr).unwrap(),+ }+ }++ pub fn repeat<const OUT_DIMS: usize>(&self, rep: &Tensor<OUT_DIMS>) -> Tensor<OUT_DIMS>+ where+ Dims<OUT_DIMS>: ValidDims,+ Dims<OUT_DIMS>: DimsGt<DIMS>,+ {+ let ptr =+ unsafe { ggml_sys::ggml_repeat(self.ctx.as_ptr(), self.ptr.as_ptr(), rep.as_ptr()) };++ Tensor {+ ctx: Arc::clone(&self.ctx),+ type_: self.type_,+ ptr: NonNull::new(ptr).unwrap(),+ }+ }++ pub fn reshape<const OUT_DIMS: usize>(&self, shape: [u64; OUT_DIMS]) -> Tensor<OUT_DIMS>+ where+ Dims<OUT_DIMS>: ValidDims,+ {+ debug_assert_eq!(+ shape.into_iter().product::<u64>(),+ self.shape().into_iter().product::<u64>(),+ "reshape must not change the number of elements"+ );++ let ptr = match OUT_DIMS {+ 1 => unsafe {+ ggml_sys::ggml_reshape_1d(self.ctx.as_ptr(), self.ptr.as_ptr(), shape[0] as i64)+ },+ 2 => unsafe {+ ggml_sys::ggml_reshape_2d(+ self.ctx.as_ptr(),+ self.ptr.as_ptr(),+ shape[0] as i64,+ shape[1] as i64,+ )+ },+ 3 => unsafe {+ ggml_sys::ggml_reshape_3d(+ self.ctx.as_ptr(),+ self.ptr.as_ptr(),+ shape[0] as i64,+ shape[1] as i64,+ shape[2] as i64,+ )+ },+ 4 => unsafe {+ ggml_sys::ggml_reshape_4d(+ self.ctx.as_ptr(),+ self.ptr.as_ptr(),+ shape[0] as i64,+ shape[1] as i64,+ shape[2] as i64,+ shape[3] as i64,+ )+ },+ _ => unreachable!("{} is not a valid dimension", DIMS),+ };++ Tensor {+ ctx: Arc::clone(&self.ctx),+ type_: self.type_,+ ptr: NonNull::new(ptr).unwrap(),+ }+ }++ pub fn permute<const OUT_DIMS: usize>(&self, shape: [u64; OUT_DIMS]) -> Tensor<OUT_DIMS>+ where+ Dims<OUT_DIMS>: ValidDims,+ {+ let ptr = match OUT_DIMS {+ 1 => unsafe {+ ggml_sys::ggml_permute(+ self.ctx.as_ptr(),+ self.ptr.as_ptr(),+ shape[0] as i32,+ 1,+ 2,+ 3,+ )+ },+ 2 => unsafe {+ ggml_sys::ggml_permute(+ self.ctx.as_ptr(),+ self.ptr.as_ptr(),+ shape[0] as i32,+ shape[1] as i32,+ 2,+ 3,+ )+ },+ 3 => unsafe {+ ggml_sys::ggml_permute(+ self.ctx.as_ptr(),+ self.ptr.as_ptr(),+ shape[0] as i32,+ shape[1] as i32,+ shape[2] as i32,+ 3,+ )+ },+ 4 => unsafe {+ ggml_sys::ggml_permute(+ self.ctx.as_ptr(),+ self.ptr.as_ptr(),+ shape[0] as i32,+ shape[1] as i32,+ shape[2] as i32,+ shape[3] as i32,+ )+ },+ _ => unreachable!("{} is not a valid dimension", DIMS),+ };++ Tensor {+ ctx: Arc::clone(&self.ctx),+ type_: self.type_,+ ptr: NonNull::new(ptr).unwrap(),+ }+ }++ pub fn softmax(&self) -> Tensor<DIMS> {+ let ptr = unsafe { ggml_sys::ggml_soft_max(self.ctx.as_ptr(), self.ptr.as_ptr()) };++ Tensor {+ ctx: Arc::clone(&self.ctx),+ type_: self.type_,+ ptr: NonNull::new(ptr).unwrap(),+ }+ }++ pub fn scale(&self, scale: f32) -> Tensor<DIMS> {+ let ptr = unsafe { ggml_sys::ggml_scale(self.ctx.as_ptr(), self.ptr.as_ptr(), scale) };++ Tensor {+ ctx: Arc::clone(&self.ctx),+ type_: self.type_,+ ptr: NonNull::new(ptr).unwrap(),+ }+ }++ pub fn contiguous(&self) -> Tensor<DIMS> {+ let ptr = unsafe { ggml_sys::ggml_cont(self.ctx.as_ptr(), self.ptr.as_ptr()) };++ Tensor {+ ctx: Arc::clone(&self.ctx),+ type_: self.type_,+ ptr: NonNull::new(ptr).unwrap(),+ }+ }}macro_rules! std_ops_impl {($trait:ident, $std_func:ident, $func:ident) => {impl<const DIMS: usize> std::ops::$trait for Tensor<DIMS>where- Dims<DIMS>: ValidDims<DIMS>,+ Dims<DIMS>: ValidDims,{type Output = Tensor<DIMS>;fn $std_func(self, rhs: Self) -> Self::Output {if self.type_ != rhs.type_ {panic!("tensor types do not match");}- if self.shape != rhs.shape {- panic!("tensor shapes do not match");++ let ptr = unsafe {+ ggml_sys::$func(self.ctx.as_ptr(), self.ptr.as_ptr(), rhs.ptr.as_ptr())+ };++ Self::Output {+ ctx: Arc::clone(&self.ctx),+ type_: self.type_,+ ptr: NonNull::new(ptr).unwrap(),+ }+ }+ }++ impl<'a, 'b, const DIMS: usize> std::ops::$trait<&'b Tensor<DIMS>> for &'a Tensor<DIMS>+ where+ Dims<DIMS>: ValidDims,+ {+ type Output = Tensor<DIMS>;+ fn $std_func(self, rhs: &'b Tensor<DIMS>) -> Self::Output {+ if self.type_ != rhs.type_ {+ panic!("tensor types do not match");}let ptr = unsafe {ggml_sys::$func(self.ctx.as_ptr(), self.ptr.as_ptr(), rhs.ptr.as_ptr())};- Self {+ Self::Output {ctx: Arc::clone(&self.ctx),type_: self.type_,- shape: self.shape,ptr: NonNull::new(ptr).unwrap(),}}@@ -190,7 +390,7 @@ macro_rules! std_ops_impl {std_ops_impl!(Add, add, ggml_add);std_ops_impl!(Sub, sub, ggml_sub);-std_ops_impl!(Mul, mul, ggml_mul);+std_ops_impl!(Mul, mul, ggml_mul_mat);// impl<const DIMS: usize> Tensor<DIMS> {// pub fn concat(&self, other: &Tensor<DIMS>) -> Tensor<DIMS+1> {@@ -240,6 +440,6 @@ mod tests {}test_1d!(add_1d, add, [1.0, 2.0], [2.0, 2.0], [3.0, 4.0]);- test_1d!(mul_1d, mul, [2.0, 5.0], [2.0, 3.0], [4.0, 15.0]);+ test_1d!(mul_1d, mul, [2.0, 5.0], [2.0, 3.0], [19.0]);test_1d!(sub_1d, sub, [3.0, 10.0], [2.0, 4.0], [1.0, 6.0]);}diff --git a/frontend/src/routes/webmasters/README.md b/frontend/src/routes/webmasters/README.mdindex d833c40d..b1c4ac37 100644--- a/frontend/src/routes/webmasters/README.md+++ b/frontend/src/routes/webmasters/README.md@@ -1,7 +1,7 @@# Stract CrawlerStract is an [open source](https://github.com/StractOrg/stract/) web search engine. StractBot is the name of our crawler that collects pages from the web in order to build the index.-It is written in Rust and the source code can be found [here](https://github.com/StractOrg/stract/tree/main/core/src/crawler).+It is written in Rust and the source code can be found [here](https://github.com/StractOrg/stract/tree/main/crates/core/src/crawler).The crawler uses the user agent `Mozilla/5.0 (compatible; StractBot/0.1; open source search engine; +https://stract.com/webmasters)`.## Politenessdiff --git a/scripts/convert_hf_to_gguf.py b/scripts/convert_hf_to_gguf.pynew file mode 100755index 00000000..63eea815--- /dev/null+++ b/scripts/convert_hf_to_gguf.py@@ -0,0 +1,170 @@+#!/usr/bin/env python3++from __future__ import annotations++import argparse+import contextlib+import json+import sys+from pathlib import Path+from typing import Any, ContextManager, Iterator, cast++import numpy as np+import torch++import gguf++class Model:+ def __init__(self, dir_model: Path, ftype: int, fname_out: Path):+ self.dir_model = dir_model+ self.ftype = ftype+ self.fname_out = fname_out+ self.is_safetensors = self._is_model_safetensors()+ self.model_name = "model.safetensors" if self.is_safetensors else "pytorch_model.bin"+ self.hparams = Model.load_hparams(self.dir_model)+ self.gguf_writer = gguf.GGUFWriter(fname_out, gguf.MODEL_ARCH_NAMES[gguf.MODEL_ARCH.BERT], endianess=gguf.GGUFEndian.LITTLE, use_temp_file=False)++ def get_tensors(self) -> Iterator[tuple[str, torch.Tensor]]:+ ctx: ContextManager[Any]+ if self.is_safetensors:+ from safetensors import safe_open+ ctx = cast(ContextManager[Any], safe_open(self.dir_model / self.model_name, framework="pt", device="cpu"))+ else:+ ctx = contextlib.nullcontext(torch.load(str(self.dir_model / self.model_name), map_location="cpu", weights_only=True))++ with ctx as model_part:+ for name in model_part.keys():+ data = model_part.get_tensor(name) if self.is_safetensors else model_part[name]+ yield name, data++ def set_gguf_parameters(self):+ self.gguf_writer.add_name(self.dir_model.name)+ self.gguf_writer.add_block_count(self.hparams.get(+ "n_layers", self.hparams.get("num_hidden_layers", self.hparams.get("n_layer")),+ ))+ if (n_ctx := self.hparams.get("max_position_embeddings")) is not None:+ self.gguf_writer.add_context_length(n_ctx)+ if (n_embd := self.hparams.get("hidden_size")) is not None:+ self.gguf_writer.add_embedding_length(n_embd)+ if (n_ff := self.hparams.get("intermediate_size")) is not None:+ self.gguf_writer.add_feed_forward_length(n_ff)+ if (n_head := self.hparams.get("num_attention_heads")) is not None:+ self.gguf_writer.add_head_count(n_head)+ if (n_head_kv := self.hparams.get("num_key_value_heads")) is not None:+ self.gguf_writer.add_head_count_kv(n_head_kv)++ if (n_rms_eps := self.hparams.get("rms_norm_eps")) is not None:+ self.gguf_writer.add_layer_norm_rms_eps(n_rms_eps)+ if (n_experts := self.hparams.get("num_local_experts")) is not None:+ self.gguf_writer.add_expert_count(n_experts)+ if (n_experts_used := self.hparams.get("num_experts_per_tok")) is not None:+ self.gguf_writer.add_expert_used_count(n_experts_used)++ self.gguf_writer.add_parallel_residual(self.hparams.get("use_parallel_residual", True))++ def write_tensors(self):+ for name, data_torch in self.get_tensors():+ # we don't need these+ if name.endswith((".attention.masked_bias", ".attention.bias", ".attention.rotary_emb.inv_freq")):+ continue++ old_dtype = data_torch.dtype++ # convert any unsupported data types to float32+ if data_torch.dtype not in (torch.float16, torch.float32):+ data_torch = data_torch.to(torch.float32)++ data = data_torch.squeeze().numpy()++ n_dims = len(data.shape)+ data_dtype = data.dtype++ # if f32 desired, convert any float16 to float32+ if self.ftype == 0 and data_dtype == np.float16:+ data = data.astype(np.float32)++ # TODO: Why cant we use these float16 as-is? There should be not reason to store float16 as float32+ if self.ftype == 1 and data_dtype == np.float16 and n_dims == 1:+ data = data.astype(np.float32)++ # if f16 desired, convert any float32 2-dim weight tensors to float16+ if self.ftype == 1 and data_dtype == np.float32 and name.endswith(".weight") and n_dims == 2:+ data = data.astype(np.float16)++ print(f"{name}, n_dims = {n_dims}, {old_dtype} --> {data.dtype}")++ self.gguf_writer.add_tensor(name, data)++ def write(self):+ self.write_tensors()+ self.gguf_writer.write_header_to_file()+ self.gguf_writer.write_kv_data_to_file()+ self.gguf_writer.write_tensors_to_file()+ self.gguf_writer.close()++ @staticmethod+ def load_hparams(dir_model):+ with open(dir_model / "config.json", "r", encoding="utf-8") as f:+ return json.load(f)++ def _is_model_safetensors(self) -> bool:+ return (self.dir_model / "model.safetensors").exists()+++def parse_args() -> argparse.Namespace:+ parser = argparse.ArgumentParser(+ description="Convert a huggingface model to a GGML compatible file")+ parser.add_argument(+ "--outtype", type=str, choices=["f32", "f16"], default="f16",+ help="output format - use f32 for float32, f16 for float16",+ )+ parser.add_argument(+ "model", type=Path,+ help="directory containing model file",+ )+ parser.add_argument(+ "outfile", type=Path,+ help="path to write to",+ )++ return parser.parse_args()+++def main() -> None:+ args = parse_args()++ dir_model = args.model++ if not dir_model.is_dir():+ print(f'Error: {args.model} is not a directory', file=sys.stderr)+ sys.exit(1)++ ftype_map = {+ "f32": gguf.GGMLQuantizationType.F32,+ "f16": gguf.GGMLQuantizationType.F16,+ }++ if args.outfile is not None:+ fname_out = args.outfile+ else:+ # output in the same directory as the model by default+ fname_out = dir_model / f'ggml-model-{args.outtype}.gguf'++ print(f"Loading model: {dir_model.name}")++ hparams = Model.load_hparams(dir_model)++ with torch.inference_mode():+ model_instance = Model(dir_model, ftype_map[args.outtype], fname_out)++ print("Set model parameters")+ model_instance.set_gguf_parameters()++ print(f"Exporting model to '{fname_out}'")+ model_instance.write()++ print(f"Model successfully exported to '{fname_out}'")+++if __name__ == '__main__':+ main()diff --git a/scripts/download_hf.py b/scripts/download_hf.pynew file mode 100644index 00000000..114ee146--- /dev/null+++ b/scripts/download_hf.py@@ -0,0 +1,20 @@+from huggingface_hub import snapshot_download+import argparse++parser = argparse.ArgumentParser(description="Download a huggingface model to a local directory")++parser.add_argument(+ "local_dir", type=str+)++parser.add_argument(+ "repo_id", type=str+)++args = parser.parse_args()++snapshot_download(+ repo_id=args.repo_id,+ local_dir=args.local_dir,+ local_dir_use_symlinks=False,+ revision="main")diff --git a/scripts/download_libtorch.py b/scripts/download_libtorch.pyindex a8c2e47c..bc2b68ae 100644--- a/scripts/download_libtorch.py+++ b/scripts/download_libtorch.py@@ -68,10 +68,10 @@ def install_libtorch(gpu=False):def create_symlinks():# Create symlinks from libtorch to core/libtorch- if os.path.exists("./core/libtorch"):+ if os.path.exists("./crates/core/libtorch"):return- os.symlink("../libtorch", "./core/libtorch")+ os.symlink("../libtorch", "./crates/core/libtorch")if __name__ == "__main__":diff --git a/scripts/requirements.txt b/scripts/requirements.txtindex eabeb7e7..3766728c 100644--- a/scripts/requirements.txt+++ b/scripts/requirements.txt@@ -6,4 +6,6 @@ gensimsafetensorspandasllama-cpp-python[server]-llama-cpp-python\ No newline at end of file+llama-cpp-python+huggingface-hub>=0.20.3+gguf