Skip to content

web search done right

implement some layers in ggml linear, embedding and multihead attention

Mikkel Denkerea3b7a409927
diff --git a/.vscode/settings.json b/.vscode/settings.json
index 30cd3a46..d10c3861 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -15,14 +15,12 @@
"editor.defaultFormatter": "esbenp.prettier-vscode"
},
"prettier.configPath": "frontend/.prettierrc",
- "prettier.documentSelectors": [
- "**/*.svelte"
- ],
+ "prettier.documentSelectors": ["**/*.svelte"],
"svelte.enable-ts-plugin": true,
"tailwindCSS.experimental.configFile": "frontend/tailwind.config.ts",
"rust-analyzer.linkedProjects": [
"Cargo.toml",
"optics-lsp/Cargo.toml",
- "./core/Cargo.toml"
+ "./crates/core/Cargo.toml"
]
}
diff --git a/crates/ggml/src/context.rs b/crates/ggml/src/context.rs
index a65b62d1..18f5b53c 100644
--- a/crates/ggml/src/context.rs
+++ b/crates/ggml/src/context.rs
@@ -52,7 +52,7 @@ impl InnerGraph {
pub struct Graph<const OUT_DIMS: usize>
where
- Dims<OUT_DIMS>: ValidDims<OUT_DIMS>,
+ Dims<OUT_DIMS>: ValidDims,
{
inner: InnerGraph,
out: Tensor<OUT_DIMS>,
@@ -60,7 +60,7 @@ where
impl<const OUT_DIMS: usize> Graph<OUT_DIMS>
where
- Dims<OUT_DIMS>: ValidDims<OUT_DIMS>,
+ Dims<OUT_DIMS>: ValidDims,
{
fn inner(&self) -> &InnerGraph {
&self.inner
@@ -99,7 +99,7 @@ impl Context {
pub fn build_graph<const OUT_DIMS: usize>(&self, out: Tensor<OUT_DIMS>) -> Graph<OUT_DIMS>
where
- Dims<OUT_DIMS>: ValidDims<OUT_DIMS>,
+ Dims<OUT_DIMS>: ValidDims,
{
Graph::build(self, out)
}
@@ -110,7 +110,7 @@ impl Context {
pub fn compute<const OUT_DIMS: usize>(&self, graph: &mut Graph<OUT_DIMS>)
where
- Dims<OUT_DIMS>: ValidDims<OUT_DIMS>,
+ Dims<OUT_DIMS>: ValidDims,
{
unsafe {
ggml_sys::ggml_graph_compute_with_ctx(
diff --git a/crates/ggml/src/dims.rs b/crates/ggml/src/dims.rs
index 7e42b049..e498527c 100644
--- a/crates/ggml/src/dims.rs
+++ b/crates/ggml/src/dims.rs
@@ -1,27 +1,27 @@
pub struct Dims<const DIMS: usize>
where
- Self: ValidDims<DIMS>, {}
+ Self: ValidDims, {}
-pub trait ValidDims<const DIMS: usize> {}
+pub trait ValidDims {}
macro_rules! valid_dims_impl {
($($dims:literal)*) => {
$(
- impl ValidDims<$dims> for Dims<$dims> {}
+ impl ValidDims for Dims<$dims> {}
)*
};
}
valid_dims_impl!(1 2 3 4);
-trait DimsGt<const LHS: usize> {}
+pub trait DimsGt<const LHS: usize> {}
macro_rules! dims_gt_impl {
($($lhs:literal, $rhs:literal)*) => {
$(
impl DimsGt<$lhs> for Dims<$rhs>
where
- Dims<$rhs>: ValidDims<$rhs>,
+ Dims<$rhs>: ValidDims,
{}
)*
};
@@ -35,14 +35,14 @@ dims_gt_impl!(2, 3);
dims_gt_impl!(2, 4);
dims_gt_impl!(3, 4);
-trait DimsLt<const LHS: usize> {}
+pub trait DimsLt<const LHS: usize> {}
macro_rules! dims_lt_impl {
($($lhs:literal, $rhs:literal)*) => {
$(
impl DimsLt<$lhs> for Dims<$rhs>
where
- Dims<$rhs>: ValidDims<$rhs>,
+ Dims<$rhs>: ValidDims,
{}
)*
};
@@ -58,6 +58,24 @@ dims_lt_impl!(3, 3);
dims_lt_impl!(4, 3);
dims_lt_impl!(4, 4);
-trait DimsEq<const LHS: usize> {}
+pub trait DimsEq<const LHS: usize> {}
-impl<const DIMS: usize> DimsEq<DIMS> for Dims<DIMS> where Dims<DIMS>: ValidDims<DIMS> {}
+impl<const DIMS: usize> DimsEq<DIMS> for Dims<DIMS> where Dims<DIMS>: ValidDims {}
+
+pub trait DimsPlusOne<const N: usize> {}
+
+macro_rules! dims_plus_one_impl {
+ ($($n:literal, $res:literal)*) => {
+ $(
+ impl DimsPlusOne<$n> for Dims<$res>
+ where
+ Dims<$n>: ValidDims,
+ Dims<$res>: ValidDims,
+ {}
+ )*
+ };
+}
+
+dims_plus_one_impl!(1, 2);
+dims_plus_one_impl!(2, 3);
+dims_plus_one_impl!(3, 4);
diff --git a/crates/ggml/src/lib.rs b/crates/ggml/src/lib.rs
index 54a50c5f..2c2f7755 100644
--- a/crates/ggml/src/lib.rs
+++ b/crates/ggml/src/lib.rs
@@ -1,6 +1,7 @@
mod context;
mod dims;
-mod tensor;
+pub mod nn;
+pub mod tensor;
pub use context::*;
pub use dims::*;
diff --git a/crates/ggml/src/nn/attn.rs b/crates/ggml/src/nn/attn.rs
new file mode 100644
index 00000000..baf15b16
--- /dev/null
+++ b/crates/ggml/src/nn/attn.rs
@@ -0,0 +1,75 @@
+use super::Linear;
+use crate::{Context, GgmlType, Tensor};
+
+pub struct MultiheadAttention {
+ query: Linear,
+ key: Linear,
+ value: Linear,
+ n_embed: u64,
+ n_heads: u64,
+ head_size: u64,
+}
+impl MultiheadAttention {
+ pub fn new(ctx: &mut Context, t: GgmlType, n_embed: u64, n_heads: u64) -> Self {
+ let head_size = n_embed / n_heads;
+ let head_emb = n_heads * head_size;
+
+ let query = Linear::new(ctx, t, n_embed, head_emb);
+ let key = Linear::new(ctx, t, n_embed, head_emb);
+ let value = Linear::new(ctx, t, n_embed, head_emb);
+
+ Self {
+ query,
+ key,
+ value,
+ n_embed,
+ n_heads,
+ head_size,
+ }
+ }
+
+ pub fn forward(&self, input: &Tensor<3>) -> Tensor<3> {
+ let toks = input.shape()[1];
+ let batch_size = input.shape()[2];
+ let head_shape = [self.head_size, self.n_heads, toks, batch_size];
+
+ let query = self
+ .query
+ .forward(input)
+ .reshape(head_shape)
+ .permute([0, 2, 1, 3]);
+
+ let key = self
+ .key
+ .forward(input)
+ .reshape(head_shape)
+ .permute([0, 2, 1, 3]);
+
+ let value = self
+ .value
+ .forward(input)
+ .reshape(head_shape)
+ .permute([0, 2, 1, 3]);
+
+ let kq = (key * query)
+ .scale(1.0 / (self.n_embed as f32).sqrt())
+ .softmax();
+
+ let kqv = value.permute([1, 0, 2, 3]).contiguous() * kq;
+ let out_dims = [self.n_heads * self.head_size, toks, batch_size];
+
+ kqv.contiguous().reshape(out_dims)
+ }
+}
+#[cfg(test)]
+mod tests {
+ use super::*;
+ #[test]
+ fn test_attention() {
+ let mut ctx = Context::new(128 * 1024 * 1024, 1);
+ let input = Tensor::new(&mut ctx, GgmlType::F32, [128, 3, 5]);
+ let attention = MultiheadAttention::new(&mut ctx, GgmlType::F32, 128, 12);
+ let output = attention.forward(&input);
+ assert_eq!(output.shape(), [120, 3, 5]);
+ }
+}
diff --git a/crates/ggml/src/nn/embedding.rs b/crates/ggml/src/nn/embedding.rs
new file mode 100644
index 00000000..4629b8b0
--- /dev/null
+++ b/crates/ggml/src/nn/embedding.rs
@@ -0,0 +1,35 @@
+use crate::{Context, GgmlType, Tensor};
+
+pub struct Embedding {
+ tensor: Tensor<2>,
+}
+
+impl Embedding {
+ pub fn new(ctx: &mut Context, t: GgmlType, vocab_size: u64, embedding_dim: u64) -> Self {
+ let tensor = Tensor::new(ctx, t, [embedding_dim, vocab_size]);
+
+ Self { tensor }
+ }
+
+ pub fn forward(&mut self, input: &Tensor<1>) -> Tensor<2> {
+ self.tensor.get_rows(input)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_embedding() {
+ let mut ctx = Context::new(128 * 1024 * 1024, 1);
+
+ let mut embedding = Embedding::new(&mut ctx, GgmlType::F32, 10, 128);
+
+ let input = Tensor::new(&mut ctx, GgmlType::I32, [10]);
+
+ let output = embedding.forward(&input);
+
+ assert_eq!(output.shape(), [128, 10]);
+ }
+}
diff --git a/crates/ggml/src/nn/linear.rs b/crates/ggml/src/nn/linear.rs
new file mode 100644
index 00000000..f3dbdde7
--- /dev/null
+++ b/crates/ggml/src/nn/linear.rs
@@ -0,0 +1,38 @@
+use crate::{Context, GgmlType, Tensor};
+
+pub struct Linear {
+ weight: Tensor<3>,
+ bias: Option<Tensor<2>>,
+}
+
+impl Linear {
+ pub fn new(ctx: &mut Context, t: GgmlType, input_size: u64, output_size: u64) -> Self {
+ let weight = Tensor::new(ctx, t, [input_size, output_size, 1]);
+
+ Self { weight, bias: None }
+ }
+
+ pub fn forward(&self, input: &Tensor<3>) -> Tensor<3> {
+ let res = &self.weight * input;
+
+ if let Some(bias) = &self.bias {
+ &res + &bias.repeat(&res)
+ } else {
+ res
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_linear() {
+ let mut ctx = Context::new(128 * 1024 * 1024, 1);
+ let input = Tensor::new(&mut ctx, GgmlType::F32, [128, 3, 5]);
+ let linear = Linear::new(&mut ctx, GgmlType::F32, 128, 256);
+ let output = linear.forward(&input);
+ assert_eq!(output.shape(), [256, 3, 5]);
+ }
+}
diff --git a/crates/ggml/src/nn/mod.rs b/crates/ggml/src/nn/mod.rs
new file mode 100644
index 00000000..009d40d6
--- /dev/null
+++ b/crates/ggml/src/nn/mod.rs
@@ -0,0 +1,7 @@
+mod attn;
+mod embedding;
+mod linear;
+
+pub use attn::*;
+pub use embedding::*;
+pub use linear::*;
diff --git a/crates/ggml/src/tensor.rs b/crates/ggml/src/tensor.rs
index e5c22f4f..80797d63 100644
--- a/crates/ggml/src/tensor.rs
+++ b/crates/ggml/src/tensor.rs
@@ -2,7 +2,7 @@ use std::ptr::NonNull;
use std::sync::Arc;
use crate::context::InnerContext;
-use crate::{Context, Dims, ValidDims};
+use crate::{Context, Dims, DimsGt, DimsPlusOne, ValidDims};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[allow(non_camel_case_types)]
@@ -36,39 +36,38 @@ impl GgmlType {
GgmlType::F16 => 1,
GgmlType::Q4_0 => 2,
GgmlType::Q4_1 => 3,
- GgmlType::Q5_0 => todo!(),
- GgmlType::Q5_1 => todo!(),
- GgmlType::Q8_0 => todo!(),
- GgmlType::Q8_1 => todo!(),
- GgmlType::Q2_K => todo!(),
- GgmlType::Q3_K => todo!(),
- GgmlType::Q4_K => todo!(),
- GgmlType::Q5_K => todo!(),
- GgmlType::Q6_K => todo!(),
- GgmlType::Q8_K => todo!(),
- GgmlType::IQ2_XXS => todo!(),
- GgmlType::IQ2_XS => todo!(),
- GgmlType::I8 => todo!(),
- GgmlType::I16 => todo!(),
- GgmlType::I32 => todo!(),
- GgmlType::COUNT => todo!(),
+ GgmlType::Q5_0 => 6,
+ GgmlType::Q5_1 => 7,
+ GgmlType::Q8_0 => 8,
+ GgmlType::Q8_1 => 9,
+ GgmlType::Q2_K => 10,
+ GgmlType::Q3_K => 11,
+ GgmlType::Q4_K => 12,
+ GgmlType::Q5_K => 13,
+ GgmlType::Q6_K => 14,
+ GgmlType::Q8_K => 15,
+ GgmlType::IQ2_XXS => 16,
+ GgmlType::IQ2_XS => 17,
+ GgmlType::I8 => 18,
+ GgmlType::I16 => 19,
+ GgmlType::I32 => 20,
+ GgmlType::COUNT => 21,
}
}
}
pub struct Tensor<const DIMS: usize>
where
- Dims<DIMS>: ValidDims<DIMS>,
+ Dims<DIMS>: ValidDims,
{
ctx: Arc<InnerContext>,
type_: GgmlType,
ptr: NonNull<ggml_sys::ggml_tensor>,
- shape: [u64; DIMS],
}
impl Tensor<1> {
pub fn copy_from_slice(&mut self, slice: &[f32]) {
- if slice.len() != self.shape[0] as usize {
+ if slice.len() != self.shape()[0] as usize {
panic!("slice length does not match tensor shape");
}
@@ -83,7 +82,7 @@ impl Tensor<1> {
}
pub fn copy_to_slice(&self, out: &mut [f32]) {
- if out.len() != self.shape[0] as usize {
+ if out.len() != self.shape()[0] as usize {
panic!("slice length does not match tensor shape");
}
@@ -94,7 +93,7 @@ impl Tensor<1> {
let data = unsafe {
std::slice::from_raw_parts(
self.ptr.as_ptr().as_ref().unwrap().data as *const f32,
- self.shape[0] as usize,
+ self.shape()[0] as usize,
)
};
@@ -104,7 +103,7 @@ impl Tensor<1> {
impl<const DIMS: usize> Tensor<DIMS>
where
- Dims<DIMS>: ValidDims<DIMS>,
+ Dims<DIMS>: ValidDims,
{
pub fn new(ctx: &mut Context, t: GgmlType, shape: [u64; DIMS]) -> Self {
let ptr = match DIMS {
@@ -148,7 +147,6 @@ where
Self {
ctx: Arc::clone(ctx.inner_ctx()),
type_: t,
- shape,
ptr: NonNull::new(ptr).unwrap(),
}
}
@@ -156,31 +154,233 @@ where
pub(crate) fn as_ptr(&self) -> *mut ggml_sys::ggml_tensor {
self.ptr.as_ptr()
}
+
+ pub fn shape(&self) -> [u64; DIMS] {
+ let shape = unsafe { self.ptr.as_ptr().as_ref().unwrap().ne };
+
+ let mut out = [0; DIMS];
+
+ for i in 0..DIMS {
+ out[i] = shape[i] as u64;
+ }
+
+ out
+ }
+
+ pub fn get_rows<const ROW_DIMS: usize, const OUT_DIMS: usize>(
+ &self,
+ rows: &Tensor<ROW_DIMS>,
+ ) -> Tensor<OUT_DIMS>
+ where
+ Dims<ROW_DIMS>: ValidDims,
+ Dims<OUT_DIMS>: DimsPlusOne<ROW_DIMS> + ValidDims,
+ {
+ assert_eq!(rows.type_, GgmlType::I32, "rows tensor must be of type I32");
+
+ let ptr = unsafe {
+ ggml_sys::ggml_get_rows(self.ctx.as_ptr(), self.ptr.as_ptr(), rows.ptr.as_ptr())
+ };
+
+ Tensor {
+ ctx: Arc::clone(&self.ctx),
+ type_: self.type_,
+ ptr: NonNull::new(ptr).unwrap(),
+ }
+ }
+
+ pub fn repeat<const OUT_DIMS: usize>(&self, rep: &Tensor<OUT_DIMS>) -> Tensor<OUT_DIMS>
+ where
+ Dims<OUT_DIMS>: ValidDims,
+ Dims<OUT_DIMS>: DimsGt<DIMS>,
+ {
+ let ptr =
+ unsafe { ggml_sys::ggml_repeat(self.ctx.as_ptr(), self.ptr.as_ptr(), rep.as_ptr()) };
+
+ Tensor {
+ ctx: Arc::clone(&self.ctx),
+ type_: self.type_,
+ ptr: NonNull::new(ptr).unwrap(),
+ }
+ }
+
+ pub fn reshape<const OUT_DIMS: usize>(&self, shape: [u64; OUT_DIMS]) -> Tensor<OUT_DIMS>
+ where
+ Dims<OUT_DIMS>: ValidDims,
+ {
+ debug_assert_eq!(
+ shape.into_iter().product::<u64>(),
+ self.shape().into_iter().product::<u64>(),
+ "reshape must not change the number of elements"
+ );
+
+ let ptr = match OUT_DIMS {
+ 1 => unsafe {
+ ggml_sys::ggml_reshape_1d(self.ctx.as_ptr(), self.ptr.as_ptr(), shape[0] as i64)
+ },
+ 2 => unsafe {
+ ggml_sys::ggml_reshape_2d(
+ self.ctx.as_ptr(),
+ self.ptr.as_ptr(),
+ shape[0] as i64,
+ shape[1] as i64,
+ )
+ },
+ 3 => unsafe {
+ ggml_sys::ggml_reshape_3d(
+ self.ctx.as_ptr(),
+ self.ptr.as_ptr(),
+ shape[0] as i64,
+ shape[1] as i64,
+ shape[2] as i64,
+ )
+ },
+ 4 => unsafe {
+ ggml_sys::ggml_reshape_4d(
+ self.ctx.as_ptr(),
+ self.ptr.as_ptr(),
+ shape[0] as i64,
+ shape[1] as i64,
+ shape[2] as i64,
+ shape[3] as i64,
+ )
+ },
+ _ => unreachable!("{} is not a valid dimension", DIMS),
+ };
+
+ Tensor {
+ ctx: Arc::clone(&self.ctx),
+ type_: self.type_,
+ ptr: NonNull::new(ptr).unwrap(),
+ }
+ }
+
+ pub fn permute<const OUT_DIMS: usize>(&self, shape: [u64; OUT_DIMS]) -> Tensor<OUT_DIMS>
+ where
+ Dims<OUT_DIMS>: ValidDims,
+ {
+ let ptr = match OUT_DIMS {
+ 1 => unsafe {
+ ggml_sys::ggml_permute(
+ self.ctx.as_ptr(),
+ self.ptr.as_ptr(),
+ shape[0] as i32,
+ 1,
+ 2,
+ 3,
+ )
+ },
+ 2 => unsafe {
+ ggml_sys::ggml_permute(
+ self.ctx.as_ptr(),
+ self.ptr.as_ptr(),
+ shape[0] as i32,
+ shape[1] as i32,
+ 2,
+ 3,
+ )
+ },
+ 3 => unsafe {
+ ggml_sys::ggml_permute(
+ self.ctx.as_ptr(),
+ self.ptr.as_ptr(),
+ shape[0] as i32,
+ shape[1] as i32,
+ shape[2] as i32,
+ 3,
+ )
+ },
+ 4 => unsafe {
+ ggml_sys::ggml_permute(
+ self.ctx.as_ptr(),
+ self.ptr.as_ptr(),
+ shape[0] as i32,
+ shape[1] as i32,
+ shape[2] as i32,
+ shape[3] as i32,
+ )
+ },
+ _ => unreachable!("{} is not a valid dimension", DIMS),
+ };
+
+ Tensor {
+ ctx: Arc::clone(&self.ctx),
+ type_: self.type_,
+ ptr: NonNull::new(ptr).unwrap(),
+ }
+ }
+
+ pub fn softmax(&self) -> Tensor<DIMS> {
+ let ptr = unsafe { ggml_sys::ggml_soft_max(self.ctx.as_ptr(), self.ptr.as_ptr()) };
+
+ Tensor {
+ ctx: Arc::clone(&self.ctx),
+ type_: self.type_,
+ ptr: NonNull::new(ptr).unwrap(),
+ }
+ }
+
+ pub fn scale(&self, scale: f32) -> Tensor<DIMS> {
+ let ptr = unsafe { ggml_sys::ggml_scale(self.ctx.as_ptr(), self.ptr.as_ptr(), scale) };
+
+ Tensor {
+ ctx: Arc::clone(&self.ctx),
+ type_: self.type_,
+ ptr: NonNull::new(ptr).unwrap(),
+ }
+ }
+
+ pub fn contiguous(&self) -> Tensor<DIMS> {
+ let ptr = unsafe { ggml_sys::ggml_cont(self.ctx.as_ptr(), self.ptr.as_ptr()) };
+
+ Tensor {
+ ctx: Arc::clone(&self.ctx),
+ type_: self.type_,
+ ptr: NonNull::new(ptr).unwrap(),
+ }
+ }
}
macro_rules! std_ops_impl {
($trait:ident, $std_func:ident, $func:ident) => {
impl<const DIMS: usize> std::ops::$trait for Tensor<DIMS>
where
- Dims<DIMS>: ValidDims<DIMS>,
+ Dims<DIMS>: ValidDims,
{
type Output = Tensor<DIMS>;
fn $std_func(self, rhs: Self) -> Self::Output {
if self.type_ != rhs.type_ {
panic!("tensor types do not match");
}
- if self.shape != rhs.shape {
- panic!("tensor shapes do not match");
+
+ let ptr = unsafe {
+ ggml_sys::$func(self.ctx.as_ptr(), self.ptr.as_ptr(), rhs.ptr.as_ptr())
+ };
+
+ Self::Output {
+ ctx: Arc::clone(&self.ctx),
+ type_: self.type_,
+ ptr: NonNull::new(ptr).unwrap(),
+ }
+ }
+ }
+
+ impl<'a, 'b, const DIMS: usize> std::ops::$trait<&'b Tensor<DIMS>> for &'a Tensor<DIMS>
+ where
+ Dims<DIMS>: ValidDims,
+ {
+ type Output = Tensor<DIMS>;
+ fn $std_func(self, rhs: &'b Tensor<DIMS>) -> Self::Output {
+ if self.type_ != rhs.type_ {
+ panic!("tensor types do not match");
}
let ptr = unsafe {
ggml_sys::$func(self.ctx.as_ptr(), self.ptr.as_ptr(), rhs.ptr.as_ptr())
};
- Self {
+ Self::Output {
ctx: Arc::clone(&self.ctx),
type_: self.type_,
- shape: self.shape,
ptr: NonNull::new(ptr).unwrap(),
}
}
@@ -190,7 +390,7 @@ macro_rules! std_ops_impl {
std_ops_impl!(Add, add, ggml_add);
std_ops_impl!(Sub, sub, ggml_sub);
-std_ops_impl!(Mul, mul, ggml_mul);
+std_ops_impl!(Mul, mul, ggml_mul_mat);
// impl<const DIMS: usize> Tensor<DIMS> {
// pub fn concat(&self, other: &Tensor<DIMS>) -> Tensor<DIMS+1> {
@@ -240,6 +440,6 @@ mod tests {
}
test_1d!(add_1d, add, [1.0, 2.0], [2.0, 2.0], [3.0, 4.0]);
- test_1d!(mul_1d, mul, [2.0, 5.0], [2.0, 3.0], [4.0, 15.0]);
+ test_1d!(mul_1d, mul, [2.0, 5.0], [2.0, 3.0], [19.0]);
test_1d!(sub_1d, sub, [3.0, 10.0], [2.0, 4.0], [1.0, 6.0]);
}
diff --git a/frontend/src/routes/webmasters/README.md b/frontend/src/routes/webmasters/README.md
index d833c40d..b1c4ac37 100644
--- a/frontend/src/routes/webmasters/README.md
+++ b/frontend/src/routes/webmasters/README.md
@@ -1,7 +1,7 @@
# Stract Crawler
Stract is an [open source](https://github.com/StractOrg/stract/) web search engine. StractBot is the name of our crawler that collects pages from the web in order to build the index.
-It is written in Rust and the source code can be found [here](https://github.com/StractOrg/stract/tree/main/core/src/crawler).
+It is written in Rust and the source code can be found [here](https://github.com/StractOrg/stract/tree/main/crates/core/src/crawler).
The crawler uses the user agent `Mozilla/5.0 (compatible; StractBot/0.1; open source search engine; +https://stract.com/webmasters)`.
## Politeness
diff --git a/scripts/convert_hf_to_gguf.py b/scripts/convert_hf_to_gguf.py
new file mode 100755
index 00000000..63eea815
--- /dev/null
+++ b/scripts/convert_hf_to_gguf.py
@@ -0,0 +1,170 @@
+#!/usr/bin/env python3
+
+from __future__ import annotations
+
+import argparse
+import contextlib
+import json
+import sys
+from pathlib import Path
+from typing import Any, ContextManager, Iterator, cast
+
+import numpy as np
+import torch
+
+import gguf
+
+class Model:
+ def __init__(self, dir_model: Path, ftype: int, fname_out: Path):
+ self.dir_model = dir_model
+ self.ftype = ftype
+ self.fname_out = fname_out
+ self.is_safetensors = self._is_model_safetensors()
+ self.model_name = "model.safetensors" if self.is_safetensors else "pytorch_model.bin"
+ self.hparams = Model.load_hparams(self.dir_model)
+ self.gguf_writer = gguf.GGUFWriter(fname_out, gguf.MODEL_ARCH_NAMES[gguf.MODEL_ARCH.BERT], endianess=gguf.GGUFEndian.LITTLE, use_temp_file=False)
+
+ def get_tensors(self) -> Iterator[tuple[str, torch.Tensor]]:
+ ctx: ContextManager[Any]
+ if self.is_safetensors:
+ from safetensors import safe_open
+ ctx = cast(ContextManager[Any], safe_open(self.dir_model / self.model_name, framework="pt", device="cpu"))
+ else:
+ ctx = contextlib.nullcontext(torch.load(str(self.dir_model / self.model_name), map_location="cpu", weights_only=True))
+
+ with ctx as model_part:
+ for name in model_part.keys():
+ data = model_part.get_tensor(name) if self.is_safetensors else model_part[name]
+ yield name, data
+
+ def set_gguf_parameters(self):
+ self.gguf_writer.add_name(self.dir_model.name)
+ self.gguf_writer.add_block_count(self.hparams.get(
+ "n_layers", self.hparams.get("num_hidden_layers", self.hparams.get("n_layer")),
+ ))
+ if (n_ctx := self.hparams.get("max_position_embeddings")) is not None:
+ self.gguf_writer.add_context_length(n_ctx)
+ if (n_embd := self.hparams.get("hidden_size")) is not None:
+ self.gguf_writer.add_embedding_length(n_embd)
+ if (n_ff := self.hparams.get("intermediate_size")) is not None:
+ self.gguf_writer.add_feed_forward_length(n_ff)
+ if (n_head := self.hparams.get("num_attention_heads")) is not None:
+ self.gguf_writer.add_head_count(n_head)
+ if (n_head_kv := self.hparams.get("num_key_value_heads")) is not None:
+ self.gguf_writer.add_head_count_kv(n_head_kv)
+
+ if (n_rms_eps := self.hparams.get("rms_norm_eps")) is not None:
+ self.gguf_writer.add_layer_norm_rms_eps(n_rms_eps)
+ if (n_experts := self.hparams.get("num_local_experts")) is not None:
+ self.gguf_writer.add_expert_count(n_experts)
+ if (n_experts_used := self.hparams.get("num_experts_per_tok")) is not None:
+ self.gguf_writer.add_expert_used_count(n_experts_used)
+
+ self.gguf_writer.add_parallel_residual(self.hparams.get("use_parallel_residual", True))
+
+ def write_tensors(self):
+ for name, data_torch in self.get_tensors():
+ # we don't need these
+ if name.endswith((".attention.masked_bias", ".attention.bias", ".attention.rotary_emb.inv_freq")):
+ continue
+
+ old_dtype = data_torch.dtype
+
+ # convert any unsupported data types to float32
+ if data_torch.dtype not in (torch.float16, torch.float32):
+ data_torch = data_torch.to(torch.float32)
+
+ data = data_torch.squeeze().numpy()
+
+ n_dims = len(data.shape)
+ data_dtype = data.dtype
+
+ # if f32 desired, convert any float16 to float32
+ if self.ftype == 0 and data_dtype == np.float16:
+ data = data.astype(np.float32)
+
+ # TODO: Why cant we use these float16 as-is? There should be not reason to store float16 as float32
+ if self.ftype == 1 and data_dtype == np.float16 and n_dims == 1:
+ data = data.astype(np.float32)
+
+ # if f16 desired, convert any float32 2-dim weight tensors to float16
+ if self.ftype == 1 and data_dtype == np.float32 and name.endswith(".weight") and n_dims == 2:
+ data = data.astype(np.float16)
+
+ print(f"{name}, n_dims = {n_dims}, {old_dtype} --> {data.dtype}")
+
+ self.gguf_writer.add_tensor(name, data)
+
+ def write(self):
+ self.write_tensors()
+ self.gguf_writer.write_header_to_file()
+ self.gguf_writer.write_kv_data_to_file()
+ self.gguf_writer.write_tensors_to_file()
+ self.gguf_writer.close()
+
+ @staticmethod
+ def load_hparams(dir_model):
+ with open(dir_model / "config.json", "r", encoding="utf-8") as f:
+ return json.load(f)
+
+ def _is_model_safetensors(self) -> bool:
+ return (self.dir_model / "model.safetensors").exists()
+
+
+def parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(
+ description="Convert a huggingface model to a GGML compatible file")
+ parser.add_argument(
+ "--outtype", type=str, choices=["f32", "f16"], default="f16",
+ help="output format - use f32 for float32, f16 for float16",
+ )
+ parser.add_argument(
+ "model", type=Path,
+ help="directory containing model file",
+ )
+ parser.add_argument(
+ "outfile", type=Path,
+ help="path to write to",
+ )
+
+ return parser.parse_args()
+
+
+def main() -> None:
+ args = parse_args()
+
+ dir_model = args.model
+
+ if not dir_model.is_dir():
+ print(f'Error: {args.model} is not a directory', file=sys.stderr)
+ sys.exit(1)
+
+ ftype_map = {
+ "f32": gguf.GGMLQuantizationType.F32,
+ "f16": gguf.GGMLQuantizationType.F16,
+ }
+
+ if args.outfile is not None:
+ fname_out = args.outfile
+ else:
+ # output in the same directory as the model by default
+ fname_out = dir_model / f'ggml-model-{args.outtype}.gguf'
+
+ print(f"Loading model: {dir_model.name}")
+
+ hparams = Model.load_hparams(dir_model)
+
+ with torch.inference_mode():
+ model_instance = Model(dir_model, ftype_map[args.outtype], fname_out)
+
+ print("Set model parameters")
+ model_instance.set_gguf_parameters()
+
+ print(f"Exporting model to '{fname_out}'")
+ model_instance.write()
+
+ print(f"Model successfully exported to '{fname_out}'")
+
+
+if __name__ == '__main__':
+ main()
diff --git a/scripts/download_hf.py b/scripts/download_hf.py
new file mode 100644
index 00000000..114ee146
--- /dev/null
+++ b/scripts/download_hf.py
@@ -0,0 +1,20 @@
+from huggingface_hub import snapshot_download
+import argparse
+
+parser = argparse.ArgumentParser(description="Download a huggingface model to a local directory")
+
+parser.add_argument(
+ "local_dir", type=str
+)
+
+parser.add_argument(
+ "repo_id", type=str
+)
+
+args = parser.parse_args()
+
+snapshot_download(
+ repo_id=args.repo_id,
+ local_dir=args.local_dir,
+ local_dir_use_symlinks=False,
+ revision="main")
diff --git a/scripts/download_libtorch.py b/scripts/download_libtorch.py
index a8c2e47c..bc2b68ae 100644
--- a/scripts/download_libtorch.py
+++ b/scripts/download_libtorch.py
@@ -68,10 +68,10 @@ def install_libtorch(gpu=False):
def create_symlinks():
# Create symlinks from libtorch to core/libtorch
- if os.path.exists("./core/libtorch"):
+ if os.path.exists("./crates/core/libtorch"):
return
- os.symlink("../libtorch", "./core/libtorch")
+ os.symlink("../libtorch", "./crates/core/libtorch")
if __name__ == "__main__":
diff --git a/scripts/requirements.txt b/scripts/requirements.txt
index eabeb7e7..3766728c 100644
--- a/scripts/requirements.txt
+++ b/scripts/requirements.txt
@@ -6,4 +6,6 @@ gensim
safetensors
pandas
llama-cpp-python[server]
-llama-cpp-python
\ No newline at end of file
+llama-cpp-python
+huggingface-hub>=0.20.3
+gguf