Skip to content

web search done right

stash ggml just found the 'candle' library from huggingface written in pure rust. this would be a much better library for us to use as they already implement all the quanitization features, f16 etc. and we can be sure it's safe. unfortunately, it renders all the work i did over the past couple of days on ggml bindings useless... stashing it here before i delete it in case we need it in the future.

Mikkel Denkerc7625a042e60
diff --git a/Cargo.lock b/Cargo.lock
index c0ab5faa..102a6d18 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4300,6 +4300,7 @@ dependencies = [
"fnv",
"fst",
"futures",
+ "ggml",
"half 2.3.1",
"hashbrown 0.14.1",
"http 1.0.0",
diff --git a/configs/api.toml b/configs/api.toml
index fa50cbba..ddd27b1b 100644
--- a/configs/api.toml
+++ b/configs/api.toml
@@ -8,9 +8,6 @@ prometheus_host = "0.0.0.0:3001"
queries_csv_path = "data/queries_us.csv"
spell_checker_path = "data/web_spell/checker"
bangs_path = "data/bangs.json"
-openai_api_model = "data/mistral-7b-instruct-v0.2.Q4_K_M.gguf"
-# openai_api_model = "TheBloke/Mistral-7B-Instruct-v0.2-AWQ"
-openai_api_uri = "http://localhost:4000"
summarizer_path = "data/summarizer"
[thresholds]
@@ -23,3 +20,9 @@ thesaurus_paths = ["data/english-wordnet-2022-subset.ttl"]
[correction_config]
correction_threshold = 3.0
lm_prob_weight = 1.0
+
+[llm]
+api_base= "http://localhost:4000/v1"
+model = "data/mistral-7b-instruct-v0.2.Q4_K_M.gguf"
+# model = "TheBloke/Mistral-7B-Instruct-v0.2-AWQ"
+# model = "mistralai/Mixtral-8x7B-Instruct-v0.1"
diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml
index 2444cfd2..173cc652 100644
--- a/crates/core/Cargo.toml
+++ b/crates/core/Cargo.toml
@@ -47,6 +47,7 @@ flate2 = {workspace = true}
fnv = {workspace = true}
fst = {workspace = true}
futures = {workspace = true}
+ggml = {path = "../ggml"}
half = {workspace = true}
hashbrown = {workspace = true}
http = {workspace = true}
diff --git a/crates/core/src/api/mod.rs b/crates/core/src/api/mod.rs
index 474eff5b..f4270115 100644
--- a/crates/core/src/api/mod.rs
+++ b/crates/core/src/api/mod.rs
@@ -160,8 +160,9 @@ pub async fn router(config: &ApiConfig, counters: Counters) -> Result<Router> {
remote_webgraph,
summarizer: Arc::new(Summarizer::new(
&config.summarizer_path,
- config.openai_api_uri.clone(),
- config.openai_api_model.clone(),
+ config.llm.api_base.clone(),
+ config.llm.model.clone(),
+ config.llm.api_key.clone(),
)?),
improvement_queue: query_store_queue,
cluster,
diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rs
index f8f2965a..4f91bc6e 100644
--- a/crates/core/src/config/mod.rs
+++ b/crates/core/src/config/mod.rs
@@ -186,6 +186,13 @@ impl Default for ApiThresholds {
}
}
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct LLMConfig {
+ pub api_base: String,
+ pub model: String,
+ pub api_key: Option<String>,
+}
+
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct ApiConfig {
pub summarizer_path: String,
@@ -196,13 +203,13 @@ pub struct ApiConfig {
pub lambda_model_path: Option<String>,
pub spell_checker_path: Option<String>,
pub bangs_path: String,
- pub openai_api_uri: String,
- pub openai_api_model: String,
pub query_store_db_host: Option<String>,
pub cluster_id: String,
pub gossip_seed_nodes: Option<Vec<SocketAddr>>,
pub gossip_addr: SocketAddr,
+ pub llm: LLMConfig,
+
#[serde(default)]
pub collector: CollectorConfig,
diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs
index 2f56fbaa..5946527e 100644
--- a/crates/core/src/lib.rs
+++ b/crates/core/src/lib.rs
@@ -57,6 +57,7 @@ mod leaky_queue;
mod live_index;
mod llm_utils;
mod metrics;
+mod models;
pub mod naive_bayes;
pub mod prehashed;
mod query;
diff --git a/crates/core/src/llm_utils.rs b/crates/core/src/llm_utils.rs
index 9881cb72..ea87c4bf 100644
--- a/crates/core/src/llm_utils.rs
+++ b/crates/core/src/llm_utils.rs
@@ -35,6 +35,7 @@ impl Clone for ClonableTensor {
pub struct OpenAiApi {
api: String,
+ key: Option<String>,
top_p: f64,
temp: f64,
model: String,
@@ -65,14 +66,19 @@ impl OpenAiApi {
pub async fn generate(&self, prompt: &str) -> Result<String> {
let client = reqwest::Client::new();
- let res = client
- .post(format!("{}/v1/completions", &self.api))
+ let mut req = client.post(format!("{}/completions", &self.api));
+
+ if let Some(key) = &self.key {
+ req = req.bearer_auth(key);
+ }
+
+ let res = req
.json(&self.payload(prompt)?)
.send()
+ .await?
+ .text()
.await?;
- let res = res.text().await?;
-
let res: serde_json::Value = serde_json::from_str(&res)?;
let res = res
@@ -94,8 +100,13 @@ impl OpenAiApi {
let mut payload = self.payload(prompt)?;
payload["stream"] = serde_json::json!(true);
- Ok(client
- .post(format!("{}/v1/completions", &self.api))
+ let mut req = client.post(format!("{}/completions", &self.api));
+
+ if let Some(key) = &self.key {
+ req = req.bearer_auth(key);
+ }
+
+ Ok(req
.json(&payload)
.send()
.await?
@@ -122,6 +133,7 @@ impl OpenAiApi {
pub struct OpenAiApiBuilder {
api: String,
+ key: Option<String>,
model: String,
top_p: f64,
temp: f64,
@@ -131,6 +143,12 @@ pub struct OpenAiApiBuilder {
impl OpenAiApiBuilder {
pub fn new(api: String, model: String) -> Self {
+ let mut api = api;
+
+ if let Some(p) = api.strip_suffix('/') {
+ api = p.to_string();
+ }
+
Self {
api,
model,
@@ -138,9 +156,15 @@ impl OpenAiApiBuilder {
temp: 1.0,
max_tokens: None,
stop: vec![],
+ key: None,
}
}
+ pub fn api_key(mut self, key: String) -> Self {
+ self.key = Some(key);
+ self
+ }
+
pub fn top_p(mut self, top_p: f64) -> Self {
self.top_p = top_p;
self
@@ -166,6 +190,7 @@ impl OpenAiApiBuilder {
api: self.api,
top_p: self.top_p,
temp: self.temp,
+ key: self.key,
max_tokens: self.max_tokens,
stop: self.stop,
model: self.model,
diff --git a/crates/core/src/models/bert.rs b/crates/core/src/models/bert.rs
new file mode 100644
index 00000000..d7733cf6
--- /dev/null
+++ b/crates/core/src/models/bert.rs
@@ -0,0 +1,44 @@
+use ggml::{nn, Context, Tensor};
+
+struct BertEmbeddings {
+ word_embeddings: nn::Embedding,
+ position_embeddings: nn::Embedding,
+ token_type_embeddings: nn::Embedding,
+ layer_norm: nn::LayerNorm,
+}
+
+impl BertEmbeddings {
+ pub fn new(ctx: &mut Context, vocab_size: u64, hidden_size: u64) -> Self {
+ let word_embeddings = nn::Embedding::new(ctx, vocab_size, hidden_size);
+ let position_embeddings = nn::Embedding::new(ctx, 512, hidden_size);
+ let token_type_embeddings = nn::Embedding::new(ctx, 2, hidden_size);
+ let layer_norm = nn::LayerNorm::new(1e-12);
+
+ Self {
+ word_embeddings,
+ position_embeddings,
+ token_type_embeddings,
+ layer_norm,
+ }
+ }
+
+ pub fn forward(
+ &mut self,
+ input_ids: &Tensor<1>,
+ token_type_ids: &Tensor<1>,
+ position_ids: &Tensor<1>,
+ ) -> Tensor<2> {
+ let word_embeddings = self.word_embeddings.forward(input_ids);
+ let position_embeddings = self.position_embeddings.forward(position_ids);
+ let token_type_embeddings = self.token_type_embeddings.forward(token_type_ids);
+
+ let embeddings = word_embeddings + position_embeddings + token_type_embeddings;
+
+ self.layer_norm.forward(&embeddings)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+}
diff --git a/crates/core/src/models/mod.rs b/crates/core/src/models/mod.rs
new file mode 100644
index 00000000..f94d32f4
--- /dev/null
+++ b/crates/core/src/models/mod.rs
@@ -0,0 +1 @@
+mod bert;
diff --git a/crates/core/src/summarizer.rs b/crates/core/src/summarizer.rs
index df82bfec..82d28299 100644
--- a/crates/core/src/summarizer.rs
+++ b/crates/core/src/summarizer.rs
@@ -258,15 +258,16 @@ pub struct Summarizer {
impl Summarizer {
pub fn new<P: AsRef<Path>>(
path: P,
- openai_api_uri: String,
+ llm_api_base: String,
model_name: String,
+ api_key: Option<String>,
) -> Result<Self> {
Ok(Self {
extractive: ExtractiveSummarizer::open(
path.as_ref().join("dual_encoder").as_path(),
16,
)?,
- abstractive: AbstractiveSummarizer::new(openai_api_uri, model_name),
+ abstractive: AbstractiveSummarizer::new(llm_api_base, model_name, api_key),
})
}
@@ -362,11 +363,16 @@ const TRUNCATE_WORDS_ABSTRACTIVE: usize = 1024;
pub struct AbstractiveSummarizer {
api: String,
model: String,
+ api_key: Option<String>,
}
impl AbstractiveSummarizer {
- pub fn new(api: String, model: String) -> Self {
- Self { api, model }
+ pub fn new(api: String, model: String, api_key: Option<String>) -> Self {
+ Self {
+ api,
+ model,
+ api_key,
+ }
}
fn client(&self, max_tokens: Option<u64>) -> OpenAiApi {
@@ -375,6 +381,10 @@ impl AbstractiveSummarizer {
.temp(0.0)
.stop(vec!["</s>", "<|endoftext|>"]);
+ if let Some(api_key) = &self.api_key {
+ builder = builder.api_key(api_key.clone());
+ }
+
if let Some(max_tokens) = max_tokens {
builder = builder.max_tokens(max_tokens);
}
diff --git a/crates/ggml/src/context.rs b/crates/ggml/src/context.rs
index 18f5b53c..a1f83ab2 100644
--- a/crates/ggml/src/context.rs
+++ b/crates/ggml/src/context.rs
@@ -28,7 +28,9 @@ impl InnerContext {
impl Drop for InnerContext {
fn drop(&mut self) {
- unsafe { ggml_sys::ggml_free(self.ctx.as_ptr()) }
+ unsafe {
+ ggml_sys::ggml_free(self.ctx.as_ptr());
+ }
}
}
diff --git a/crates/ggml/src/nn/attn.rs b/crates/ggml/src/nn/attn.rs
index baf15b16..daf5fc10 100644
--- a/crates/ggml/src/nn/attn.rs
+++ b/crates/ggml/src/nn/attn.rs
@@ -1,7 +1,7 @@
use super::Linear;
-use crate::{Context, GgmlType, Tensor};
+use crate::{Context, Tensor};
-pub struct MultiheadAttention {
+pub struct MultiHeadAttention {
query: Linear,
key: Linear,
value: Linear,
@@ -9,14 +9,14 @@ pub struct MultiheadAttention {
n_heads: u64,
head_size: u64,
}
-impl MultiheadAttention {
- pub fn new(ctx: &mut Context, t: GgmlType, n_embed: u64, n_heads: u64) -> Self {
+impl MultiHeadAttention {
+ pub fn new(ctx: &mut Context, n_embed: u64, n_heads: u64) -> Self {
let head_size = n_embed / n_heads;
let head_emb = n_heads * head_size;
- let query = Linear::new(ctx, t, n_embed, head_emb);
- let key = Linear::new(ctx, t, n_embed, head_emb);
- let value = Linear::new(ctx, t, n_embed, head_emb);
+ let query = Linear::new(ctx, n_embed, head_emb);
+ let key = Linear::new(ctx, n_embed, head_emb);
+ let value = Linear::new(ctx, n_embed, head_emb);
Self {
query,
@@ -64,11 +64,13 @@ impl MultiheadAttention {
#[cfg(test)]
mod tests {
use super::*;
+ use crate::GgmlType;
+
#[test]
fn test_attention() {
let mut ctx = Context::new(128 * 1024 * 1024, 1);
- let input = Tensor::new(&mut ctx, GgmlType::F32, [128, 3, 5]);
- let attention = MultiheadAttention::new(&mut ctx, GgmlType::F32, 128, 12);
+ let input = Tensor::new(&mut ctx, GgmlType::default(), [128, 3, 5]);
+ let attention = MultiHeadAttention::new(&mut ctx, 128, 12);
let output = attention.forward(&input);
assert_eq!(output.shape(), [120, 3, 5]);
}
diff --git a/crates/ggml/src/nn/embedding.rs b/crates/ggml/src/nn/embedding.rs
index 4629b8b0..5fece377 100644
--- a/crates/ggml/src/nn/embedding.rs
+++ b/crates/ggml/src/nn/embedding.rs
@@ -5,8 +5,8 @@ pub struct Embedding {
}
impl Embedding {
- pub fn new(ctx: &mut Context, t: GgmlType, vocab_size: u64, embedding_dim: u64) -> Self {
- let tensor = Tensor::new(ctx, t, [embedding_dim, vocab_size]);
+ pub fn new(ctx: &mut Context, vocab_size: u64, embedding_dim: u64) -> Self {
+ let tensor = Tensor::new(ctx, GgmlType::default(), [embedding_dim, vocab_size]);
Self { tensor }
}
@@ -24,7 +24,7 @@ mod tests {
fn test_embedding() {
let mut ctx = Context::new(128 * 1024 * 1024, 1);
- let mut embedding = Embedding::new(&mut ctx, GgmlType::F32, 10, 128);
+ let mut embedding = Embedding::new(&mut ctx, 10, 128);
let input = Tensor::new(&mut ctx, GgmlType::I32, [10]);
diff --git a/crates/ggml/src/nn/layer_norm.rs b/crates/ggml/src/nn/layer_norm.rs
new file mode 100644
index 00000000..acc2a81d
--- /dev/null
+++ b/crates/ggml/src/nn/layer_norm.rs
@@ -0,0 +1,18 @@
+use crate::{Dims, Tensor, ValidDims};
+
+pub struct LayerNorm {
+ eps: f32,
+}
+
+impl LayerNorm {
+ pub fn new(eps: f32) -> Self {
+ Self { eps }
+ }
+
+ pub fn forward<const DIMS: usize>(&mut self, input: &Tensor<DIMS>) -> Tensor<DIMS>
+ where
+ Dims<DIMS>: ValidDims,
+ {
+ input.norm(self.eps)
+ }
+}
diff --git a/crates/ggml/src/nn/linear.rs b/crates/ggml/src/nn/linear.rs
index f3dbdde7..a7cabc61 100644
--- a/crates/ggml/src/nn/linear.rs
+++ b/crates/ggml/src/nn/linear.rs
@@ -6,8 +6,8 @@ pub struct Linear {
}
impl Linear {
- pub fn new(ctx: &mut Context, t: GgmlType, input_size: u64, output_size: u64) -> Self {
- let weight = Tensor::new(ctx, t, [input_size, output_size, 1]);
+ pub fn new(ctx: &mut Context, input_size: u64, output_size: u64) -> Self {
+ let weight = Tensor::new(ctx, GgmlType::default(), [input_size, output_size, 1]);
Self { weight, bias: None }
}
@@ -30,8 +30,8 @@ mod tests {
#[test]
fn test_linear() {
let mut ctx = Context::new(128 * 1024 * 1024, 1);
- let input = Tensor::new(&mut ctx, GgmlType::F32, [128, 3, 5]);
- let linear = Linear::new(&mut ctx, GgmlType::F32, 128, 256);
+ let input = Tensor::new(&mut ctx, GgmlType::default(), [128, 3, 5]);
+ let linear = Linear::new(&mut ctx, 128, 256);
let output = linear.forward(&input);
assert_eq!(output.shape(), [256, 3, 5]);
}
diff --git a/crates/ggml/src/nn/mod.rs b/crates/ggml/src/nn/mod.rs
index 009d40d6..72cb3ee2 100644
--- a/crates/ggml/src/nn/mod.rs
+++ b/crates/ggml/src/nn/mod.rs
@@ -1,7 +1,9 @@
mod attn;
mod embedding;
+mod layer_norm;
mod linear;
pub use attn::*;
pub use embedding::*;
+pub use layer_norm::*;
pub use linear::*;
diff --git a/crates/ggml/src/tensor.rs b/crates/ggml/src/tensor.rs
index 80797d63..b30884a6 100644
--- a/crates/ggml/src/tensor.rs
+++ b/crates/ggml/src/tensor.rs
@@ -4,9 +4,10 @@ use std::sync::Arc;
use crate::context::InnerContext;
use crate::{Context, Dims, DimsGt, DimsPlusOne, ValidDims};
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
#[allow(non_camel_case_types)]
pub enum GgmlType {
+ #[default]
F32,
F16,
Q4_0,
@@ -30,7 +31,7 @@ pub enum GgmlType {
}
impl GgmlType {
- fn as_raw(&self) -> usize {
+ fn as_raw(&self) -> u32 {
match &self {
GgmlType::F32 => 0,
GgmlType::F16 => 1,
@@ -65,42 +66,6 @@ where
ptr: NonNull<ggml_sys::ggml_tensor>,
}
-impl Tensor<1> {
- pub fn copy_from_slice(&mut self, slice: &[f32]) {
- if slice.len() != self.shape()[0] as usize {
- panic!("slice length does not match tensor shape");
- }
-
- if self.type_ != GgmlType::F32 {
- panic!("tensor type does not match slice type");
- }
-
- unsafe {
- let ptr = self.ptr.as_ptr().as_ref().unwrap().data;
- (ptr as *mut f32).copy_from_nonoverlapping(slice.as_ptr(), slice.len());
- }
- }
-
- pub fn copy_to_slice(&self, out: &mut [f32]) {
- if out.len() != self.shape()[0] as usize {
- panic!("slice length does not match tensor shape");
- }
-
- if self.type_ != GgmlType::F32 {
- panic!("tensor type does not match slice type");
- }
-
- let data = unsafe {
- std::slice::from_raw_parts(
- self.ptr.as_ptr().as_ref().unwrap().data as *const f32,
- self.shape()[0] as usize,
- )
- };
-
- out.copy_from_slice(data);
- }
-}
-
impl<const DIMS: usize> Tensor<DIMS>
where
Dims<DIMS>: ValidDims,
@@ -155,6 +120,48 @@ where
self.ptr.as_ptr()
}
+ pub fn num_bytes(&self) -> usize {
+ unsafe { ggml_sys::ggml_nbytes(self.ptr.as_ptr()) }
+ }
+
+ pub fn copy_from_slice(&mut self, slice: &[f32]) {
+ if slice.len() as u64 != self.num_elements() {
+ panic!("slice length does not match tensor shape");
+ }
+
+ if self.type_ != GgmlType::F32 {
+ panic!("tensor type does not match slice type");
+ }
+
+ unsafe {
+ let ptr = self.ptr.as_ptr().as_ref().unwrap().data;
+ (ptr as *mut f32).copy_from_nonoverlapping(slice.as_ptr(), slice.len());
+ }
+ }
+
+ pub fn copy_to_slice(&self, out: &mut [f32]) {
+ if out.len() as u64 != self.num_elements() {
+ panic!("slice length does not match tensor shape");
+ }
+
+ if self.type_ != GgmlType::F32 {
+ panic!("tensor type does not match slice type");
+ }
+
+ let data = unsafe {
+ std::slice::from_raw_parts(
+ self.ptr.as_ptr().as_ref().unwrap().data as *const f32,
+ self.num_elements() as usize,
+ )
+ };
+
+ out.copy_from_slice(data);
+ }
+
+ pub fn num_elements(&self) -> u64 {
+ self.shape().into_iter().product()
+ }
+
pub fn shape(&self) -> [u64; DIMS] {
let shape = unsafe { self.ptr.as_ptr().as_ref().unwrap().ne };
@@ -175,7 +182,7 @@ where
Dims<ROW_DIMS>: ValidDims,
Dims<OUT_DIMS>: DimsPlusOne<ROW_DIMS> + ValidDims,
{
- assert_eq!(rows.type_, GgmlType::I32, "rows tensor must be of type I32");
+ debug_assert_eq!(rows.type_, GgmlType::I32, "rows tensor must be of type I32");
let ptr = unsafe {
ggml_sys::ggml_get_rows(self.ctx.as_ptr(), self.ptr.as_ptr(), rows.ptr.as_ptr())
@@ -209,7 +216,7 @@ where
{
debug_assert_eq!(
shape.into_iter().product::<u64>(),
- self.shape().into_iter().product::<u64>(),
+ self.num_elements(),
"reshape must not change the number of elements"
);
@@ -338,6 +345,46 @@ where
ptr: NonNull::new(ptr).unwrap(),
}
}
+
+ pub fn as_type(&self, type_: GgmlType) -> Tensor<DIMS> {
+ let ptr =
+ unsafe { ggml_sys::ggml_cast(self.ctx.as_ptr(), self.ptr.as_ptr(), type_.as_raw()) };
+
+ Tensor {
+ ctx: Arc::clone(&self.ctx),
+ type_,
+ ptr: NonNull::new(ptr).unwrap(),
+ }
+ }
+
+ pub fn norm(&self, eps: f32) -> Tensor<DIMS> {
+ let ptr = unsafe { ggml_sys::ggml_norm(self.ctx.as_ptr(), self.ptr.as_ptr(), eps) };
+
+ Tensor {
+ ctx: Arc::clone(&self.ctx),
+ type_: self.type_,
+ ptr: NonNull::new(ptr).unwrap(),
+ }
+ }
+
+ pub fn load_bytes(&mut self, bytes: &[u8]) {
+ debug_assert_eq!(
+ bytes.len(),
+ self.num_bytes(),
+ "bytes must be the same size as the tensor"
+ );
+
+ unsafe {
+ debug_assert_ne!(
+ self.as_ptr().as_mut().unwrap().data,
+ std::ptr::null_mut(),
+ "tensor must be allocated"
+ );
+
+ (self.as_ptr().as_mut().unwrap().data as *mut u8)
+ .copy_from_nonoverlapping(bytes.as_ptr(), self.num_bytes());
+ }
+ }
}
macro_rules! std_ops_impl {
@@ -364,6 +411,50 @@ macro_rules! std_ops_impl {
}
}
+ impl<'a, const DIMS: usize> std::ops::$trait<Tensor<DIMS>> for &'a Tensor<DIMS>
+ where
+ Dims<DIMS>: ValidDims,
+ {
+ type Output = Tensor<DIMS>;
+ fn $std_func(self, rhs: Tensor<DIMS>) -> Self::Output {
+ if self.type_ != rhs.type_ {
+ panic!("tensor types do not match");
+ }
+
+ let ptr = unsafe {
+ ggml_sys::$func(self.ctx.as_ptr(), self.ptr.as_ptr(), rhs.ptr.as_ptr())
+ };
+
+ Self::Output {
+ ctx: Arc::clone(&self.ctx),
+ type_: self.type_,
+ ptr: NonNull::new(ptr).unwrap(),
+ }
+ }
+ }
+
+ impl<'b, const DIMS: usize> std::ops::$trait<&'b Tensor<DIMS>> for Tensor<DIMS>
+ where
+ Dims<DIMS>: ValidDims,
+ {
+ type Output = Tensor<DIMS>;
+ fn $std_func(self, rhs: &'b Tensor<DIMS>) -> Self::Output {
+ if self.type_ != rhs.type_ {
+ panic!("tensor types do not match");
+ }
+
+ let ptr = unsafe {
+ ggml_sys::$func(self.ctx.as_ptr(), self.ptr.as_ptr(), rhs.ptr.as_ptr())
+ };
+
+ Self::Output {
+ ctx: Arc::clone(&self.ctx),
+ type_: self.type_,
+ ptr: NonNull::new(ptr).unwrap(),
+ }
+ }
+ }
+
impl<'a, 'b, const DIMS: usize> std::ops::$trait<&'b Tensor<DIMS>> for &'a Tensor<DIMS>
where
Dims<DIMS>: ValidDims,
@@ -392,21 +483,6 @@ std_ops_impl!(Add, add, ggml_add);
std_ops_impl!(Sub, sub, ggml_sub);
std_ops_impl!(Mul, mul, ggml_mul_mat);
-// impl<const DIMS: usize> Tensor<DIMS> {
-// pub fn concat(&self, other: &Tensor<DIMS>) -> Tensor<DIMS+1> {
-// let ptr = unsafe {
-// ggml_sys::ggml_concat(self.ctx.as_ptr(), self.ptr.as_ptr(), other.ptr.as_ptr())
-// };
-//
-// Self {
-// ctx: Arc::clone(&self.ctx),
-// type_: self.type_,
-// shape: self.shape,
-// ptr: NonNull::new(ptr).unwrap(),
-// }
-// }
-// }
-
#[cfg(test)]
mod tests {
use super::*;
@@ -442,4 +518,18 @@ mod tests {
test_1d!(add_1d, add, [1.0, 2.0], [2.0, 2.0], [3.0, 4.0]);
test_1d!(mul_1d, mul, [2.0, 5.0], [2.0, 3.0], [19.0]);
test_1d!(sub_1d, sub, [3.0, 10.0], [2.0, 4.0], [1.0, 6.0]);
+
+ #[test]
+ fn num_bytes() {
+ let mut ctx = Context::new(128 * 1024 * 1024, 1);
+ let at = Tensor::new(&mut ctx, GgmlType::F32, [2, 3, 4]);
+ assert_eq!(at.num_bytes(), 32 * 2 * 3 * 4 / 8);
+ }
+
+ #[test]
+ fn load_bytes() {
+ let mut ctx = Context::new(128 * 1024 * 1024, 1);
+ let mut at = Tensor::new(&mut ctx, GgmlType::F32, [2]);
+ at.load_bytes(&[1, 2, 3, 4, 5, 6, 7, 8]);
+ }
}