web search done right
stash ggml just found the 'candle' library from huggingface written in pure rust. this would be a much better library for us to use as they already implement all the quanitization features, f16 etc. and we can be sure it's safe. unfortunately, it renders all the work i did over the past couple of days on ggml bindings useless... stashing it here before i delete it in case we need it in the future.
Mikkel Denkerc7625a042e60
diff --git a/Cargo.lock b/Cargo.lockindex c0ab5faa..102a6d18 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -4300,6 +4300,7 @@ dependencies = ["fnv","fst","futures",+ "ggml","half 2.3.1","hashbrown 0.14.1","http 1.0.0",diff --git a/configs/api.toml b/configs/api.tomlindex fa50cbba..ddd27b1b 100644--- a/configs/api.toml+++ b/configs/api.toml@@ -8,9 +8,6 @@ prometheus_host = "0.0.0.0:3001"queries_csv_path = "data/queries_us.csv"spell_checker_path = "data/web_spell/checker"bangs_path = "data/bangs.json"-openai_api_model = "data/mistral-7b-instruct-v0.2.Q4_K_M.gguf"-# openai_api_model = "TheBloke/Mistral-7B-Instruct-v0.2-AWQ"-openai_api_uri = "http://localhost:4000"summarizer_path = "data/summarizer"[thresholds]@@ -23,3 +20,9 @@ thesaurus_paths = ["data/english-wordnet-2022-subset.ttl"][correction_config]correction_threshold = 3.0lm_prob_weight = 1.0++[llm]+api_base= "http://localhost:4000/v1"+model = "data/mistral-7b-instruct-v0.2.Q4_K_M.gguf"+# model = "TheBloke/Mistral-7B-Instruct-v0.2-AWQ"+# model = "mistralai/Mixtral-8x7B-Instruct-v0.1"diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.tomlindex 2444cfd2..173cc652 100644--- a/crates/core/Cargo.toml+++ b/crates/core/Cargo.toml@@ -47,6 +47,7 @@ flate2 = {workspace = true}fnv = {workspace = true}fst = {workspace = true}futures = {workspace = true}+ggml = {path = "../ggml"}half = {workspace = true}hashbrown = {workspace = true}http = {workspace = true}diff --git a/crates/core/src/api/mod.rs b/crates/core/src/api/mod.rsindex 474eff5b..f4270115 100644--- a/crates/core/src/api/mod.rs+++ b/crates/core/src/api/mod.rs@@ -160,8 +160,9 @@ pub async fn router(config: &ApiConfig, counters: Counters) -> Result<Router> {remote_webgraph,summarizer: Arc::new(Summarizer::new(&config.summarizer_path,- config.openai_api_uri.clone(),- config.openai_api_model.clone(),+ config.llm.api_base.clone(),+ config.llm.model.clone(),+ config.llm.api_key.clone(),)?),improvement_queue: query_store_queue,cluster,diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rsindex f8f2965a..4f91bc6e 100644--- a/crates/core/src/config/mod.rs+++ b/crates/core/src/config/mod.rs@@ -186,6 +186,13 @@ impl Default for ApiThresholds {}}+#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct LLMConfig {+ pub api_base: String,+ pub model: String,+ pub api_key: Option<String>,+}+#[derive(Debug, Serialize, Deserialize, Clone)]pub struct ApiConfig {pub summarizer_path: String,@@ -196,13 +203,13 @@ pub struct ApiConfig {pub lambda_model_path: Option<String>,pub spell_checker_path: Option<String>,pub bangs_path: String,- pub openai_api_uri: String,- pub openai_api_model: String,pub query_store_db_host: Option<String>,pub cluster_id: String,pub gossip_seed_nodes: Option<Vec<SocketAddr>>,pub gossip_addr: SocketAddr,+ pub llm: LLMConfig,+#[serde(default)]pub collector: CollectorConfig,diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rsindex 2f56fbaa..5946527e 100644--- a/crates/core/src/lib.rs+++ b/crates/core/src/lib.rs@@ -57,6 +57,7 @@ mod leaky_queue;mod live_index;mod llm_utils;mod metrics;+mod models;pub mod naive_bayes;pub mod prehashed;mod query;diff --git a/crates/core/src/llm_utils.rs b/crates/core/src/llm_utils.rsindex 9881cb72..ea87c4bf 100644--- a/crates/core/src/llm_utils.rs+++ b/crates/core/src/llm_utils.rs@@ -35,6 +35,7 @@ impl Clone for ClonableTensor {pub struct OpenAiApi {api: String,+ key: Option<String>,top_p: f64,temp: f64,model: String,@@ -65,14 +66,19 @@ impl OpenAiApi {pub async fn generate(&self, prompt: &str) -> Result<String> {let client = reqwest::Client::new();- let res = client- .post(format!("{}/v1/completions", &self.api))+ let mut req = client.post(format!("{}/completions", &self.api));++ if let Some(key) = &self.key {+ req = req.bearer_auth(key);+ }++ let res = req.json(&self.payload(prompt)?).send()+ .await?+ .text().await?;- let res = res.text().await?;-let res: serde_json::Value = serde_json::from_str(&res)?;let res = res@@ -94,8 +100,13 @@ impl OpenAiApi {let mut payload = self.payload(prompt)?;payload["stream"] = serde_json::json!(true);- Ok(client- .post(format!("{}/v1/completions", &self.api))+ let mut req = client.post(format!("{}/completions", &self.api));++ if let Some(key) = &self.key {+ req = req.bearer_auth(key);+ }++ Ok(req.json(&payload).send().await?@@ -122,6 +133,7 @@ impl OpenAiApi {pub struct OpenAiApiBuilder {api: String,+ key: Option<String>,model: String,top_p: f64,temp: f64,@@ -131,6 +143,12 @@ pub struct OpenAiApiBuilder {impl OpenAiApiBuilder {pub fn new(api: String, model: String) -> Self {+ let mut api = api;++ if let Some(p) = api.strip_suffix('/') {+ api = p.to_string();+ }+Self {api,model,@@ -138,9 +156,15 @@ impl OpenAiApiBuilder {temp: 1.0,max_tokens: None,stop: vec![],+ key: None,}}+ pub fn api_key(mut self, key: String) -> Self {+ self.key = Some(key);+ self+ }+pub fn top_p(mut self, top_p: f64) -> Self {self.top_p = top_p;self@@ -166,6 +190,7 @@ impl OpenAiApiBuilder {api: self.api,top_p: self.top_p,temp: self.temp,+ key: self.key,max_tokens: self.max_tokens,stop: self.stop,model: self.model,diff --git a/crates/core/src/models/bert.rs b/crates/core/src/models/bert.rsnew file mode 100644index 00000000..d7733cf6--- /dev/null+++ b/crates/core/src/models/bert.rs@@ -0,0 +1,44 @@+use ggml::{nn, Context, Tensor};++struct BertEmbeddings {+ word_embeddings: nn::Embedding,+ position_embeddings: nn::Embedding,+ token_type_embeddings: nn::Embedding,+ layer_norm: nn::LayerNorm,+}++impl BertEmbeddings {+ pub fn new(ctx: &mut Context, vocab_size: u64, hidden_size: u64) -> Self {+ let word_embeddings = nn::Embedding::new(ctx, vocab_size, hidden_size);+ let position_embeddings = nn::Embedding::new(ctx, 512, hidden_size);+ let token_type_embeddings = nn::Embedding::new(ctx, 2, hidden_size);+ let layer_norm = nn::LayerNorm::new(1e-12);++ Self {+ word_embeddings,+ position_embeddings,+ token_type_embeddings,+ layer_norm,+ }+ }++ pub fn forward(+ &mut self,+ input_ids: &Tensor<1>,+ token_type_ids: &Tensor<1>,+ position_ids: &Tensor<1>,+ ) -> Tensor<2> {+ let word_embeddings = self.word_embeddings.forward(input_ids);+ let position_embeddings = self.position_embeddings.forward(position_ids);+ let token_type_embeddings = self.token_type_embeddings.forward(token_type_ids);++ let embeddings = word_embeddings + position_embeddings + token_type_embeddings;++ self.layer_norm.forward(&embeddings)+ }+}++#[cfg(test)]+mod tests {+ use super::*;+}diff --git a/crates/core/src/models/mod.rs b/crates/core/src/models/mod.rsnew file mode 100644index 00000000..f94d32f4--- /dev/null+++ b/crates/core/src/models/mod.rs@@ -0,0 +1 @@+mod bert;diff --git a/crates/core/src/summarizer.rs b/crates/core/src/summarizer.rsindex df82bfec..82d28299 100644--- a/crates/core/src/summarizer.rs+++ b/crates/core/src/summarizer.rs@@ -258,15 +258,16 @@ pub struct Summarizer {impl Summarizer {pub fn new<P: AsRef<Path>>(path: P,- openai_api_uri: String,+ llm_api_base: String,model_name: String,+ api_key: Option<String>,) -> Result<Self> {Ok(Self {extractive: ExtractiveSummarizer::open(path.as_ref().join("dual_encoder").as_path(),16,)?,- abstractive: AbstractiveSummarizer::new(openai_api_uri, model_name),+ abstractive: AbstractiveSummarizer::new(llm_api_base, model_name, api_key),})}@@ -362,11 +363,16 @@ const TRUNCATE_WORDS_ABSTRACTIVE: usize = 1024;pub struct AbstractiveSummarizer {api: String,model: String,+ api_key: Option<String>,}impl AbstractiveSummarizer {- pub fn new(api: String, model: String) -> Self {- Self { api, model }+ pub fn new(api: String, model: String, api_key: Option<String>) -> Self {+ Self {+ api,+ model,+ api_key,+ }}fn client(&self, max_tokens: Option<u64>) -> OpenAiApi {@@ -375,6 +381,10 @@ impl AbstractiveSummarizer {.temp(0.0).stop(vec!["</s>", "<|endoftext|>"]);+ if let Some(api_key) = &self.api_key {+ builder = builder.api_key(api_key.clone());+ }+if let Some(max_tokens) = max_tokens {builder = builder.max_tokens(max_tokens);}diff --git a/crates/ggml/src/context.rs b/crates/ggml/src/context.rsindex 18f5b53c..a1f83ab2 100644--- a/crates/ggml/src/context.rs+++ b/crates/ggml/src/context.rs@@ -28,7 +28,9 @@ impl InnerContext {impl Drop for InnerContext {fn drop(&mut self) {- unsafe { ggml_sys::ggml_free(self.ctx.as_ptr()) }+ unsafe {+ ggml_sys::ggml_free(self.ctx.as_ptr());+ }}}diff --git a/crates/ggml/src/nn/attn.rs b/crates/ggml/src/nn/attn.rsindex baf15b16..daf5fc10 100644--- a/crates/ggml/src/nn/attn.rs+++ b/crates/ggml/src/nn/attn.rs@@ -1,7 +1,7 @@use super::Linear;-use crate::{Context, GgmlType, Tensor};+use crate::{Context, Tensor};-pub struct MultiheadAttention {+pub struct MultiHeadAttention {query: Linear,key: Linear,value: Linear,@@ -9,14 +9,14 @@ pub struct MultiheadAttention {n_heads: u64,head_size: u64,}-impl MultiheadAttention {- pub fn new(ctx: &mut Context, t: GgmlType, n_embed: u64, n_heads: u64) -> Self {+impl MultiHeadAttention {+ pub fn new(ctx: &mut Context, n_embed: u64, n_heads: u64) -> Self {let head_size = n_embed / n_heads;let head_emb = n_heads * head_size;- let query = Linear::new(ctx, t, n_embed, head_emb);- let key = Linear::new(ctx, t, n_embed, head_emb);- let value = Linear::new(ctx, t, n_embed, head_emb);+ let query = Linear::new(ctx, n_embed, head_emb);+ let key = Linear::new(ctx, n_embed, head_emb);+ let value = Linear::new(ctx, n_embed, head_emb);Self {query,@@ -64,11 +64,13 @@ impl MultiheadAttention {#[cfg(test)]mod tests {use super::*;+ use crate::GgmlType;+#[test]fn test_attention() {let mut ctx = Context::new(128 * 1024 * 1024, 1);- let input = Tensor::new(&mut ctx, GgmlType::F32, [128, 3, 5]);- let attention = MultiheadAttention::new(&mut ctx, GgmlType::F32, 128, 12);+ let input = Tensor::new(&mut ctx, GgmlType::default(), [128, 3, 5]);+ let attention = MultiHeadAttention::new(&mut ctx, 128, 12);let output = attention.forward(&input);assert_eq!(output.shape(), [120, 3, 5]);}diff --git a/crates/ggml/src/nn/embedding.rs b/crates/ggml/src/nn/embedding.rsindex 4629b8b0..5fece377 100644--- a/crates/ggml/src/nn/embedding.rs+++ b/crates/ggml/src/nn/embedding.rs@@ -5,8 +5,8 @@ pub struct Embedding {}impl Embedding {- pub fn new(ctx: &mut Context, t: GgmlType, vocab_size: u64, embedding_dim: u64) -> Self {- let tensor = Tensor::new(ctx, t, [embedding_dim, vocab_size]);+ pub fn new(ctx: &mut Context, vocab_size: u64, embedding_dim: u64) -> Self {+ let tensor = Tensor::new(ctx, GgmlType::default(), [embedding_dim, vocab_size]);Self { tensor }}@@ -24,7 +24,7 @@ mod tests {fn test_embedding() {let mut ctx = Context::new(128 * 1024 * 1024, 1);- let mut embedding = Embedding::new(&mut ctx, GgmlType::F32, 10, 128);+ let mut embedding = Embedding::new(&mut ctx, 10, 128);let input = Tensor::new(&mut ctx, GgmlType::I32, [10]);diff --git a/crates/ggml/src/nn/layer_norm.rs b/crates/ggml/src/nn/layer_norm.rsnew file mode 100644index 00000000..acc2a81d--- /dev/null+++ b/crates/ggml/src/nn/layer_norm.rs@@ -0,0 +1,18 @@+use crate::{Dims, Tensor, ValidDims};++pub struct LayerNorm {+ eps: f32,+}++impl LayerNorm {+ pub fn new(eps: f32) -> Self {+ Self { eps }+ }++ pub fn forward<const DIMS: usize>(&mut self, input: &Tensor<DIMS>) -> Tensor<DIMS>+ where+ Dims<DIMS>: ValidDims,+ {+ input.norm(self.eps)+ }+}diff --git a/crates/ggml/src/nn/linear.rs b/crates/ggml/src/nn/linear.rsindex f3dbdde7..a7cabc61 100644--- a/crates/ggml/src/nn/linear.rs+++ b/crates/ggml/src/nn/linear.rs@@ -6,8 +6,8 @@ pub struct Linear {}impl Linear {- pub fn new(ctx: &mut Context, t: GgmlType, input_size: u64, output_size: u64) -> Self {- let weight = Tensor::new(ctx, t, [input_size, output_size, 1]);+ pub fn new(ctx: &mut Context, input_size: u64, output_size: u64) -> Self {+ let weight = Tensor::new(ctx, GgmlType::default(), [input_size, output_size, 1]);Self { weight, bias: None }}@@ -30,8 +30,8 @@ mod tests {#[test]fn test_linear() {let mut ctx = Context::new(128 * 1024 * 1024, 1);- let input = Tensor::new(&mut ctx, GgmlType::F32, [128, 3, 5]);- let linear = Linear::new(&mut ctx, GgmlType::F32, 128, 256);+ let input = Tensor::new(&mut ctx, GgmlType::default(), [128, 3, 5]);+ let linear = Linear::new(&mut ctx, 128, 256);let output = linear.forward(&input);assert_eq!(output.shape(), [256, 3, 5]);}diff --git a/crates/ggml/src/nn/mod.rs b/crates/ggml/src/nn/mod.rsindex 009d40d6..72cb3ee2 100644--- a/crates/ggml/src/nn/mod.rs+++ b/crates/ggml/src/nn/mod.rs@@ -1,7 +1,9 @@mod attn;mod embedding;+mod layer_norm;mod linear;pub use attn::*;pub use embedding::*;+pub use layer_norm::*;pub use linear::*;diff --git a/crates/ggml/src/tensor.rs b/crates/ggml/src/tensor.rsindex 80797d63..b30884a6 100644--- a/crates/ggml/src/tensor.rs+++ b/crates/ggml/src/tensor.rs@@ -4,9 +4,10 @@ use std::sync::Arc;use crate::context::InnerContext;use crate::{Context, Dims, DimsGt, DimsPlusOne, ValidDims};-#[derive(Debug, Clone, Copy, PartialEq, Eq)]+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]#[allow(non_camel_case_types)]pub enum GgmlType {+ #[default]F32,F16,Q4_0,@@ -30,7 +31,7 @@ pub enum GgmlType {}impl GgmlType {- fn as_raw(&self) -> usize {+ fn as_raw(&self) -> u32 {match &self {GgmlType::F32 => 0,GgmlType::F16 => 1,@@ -65,42 +66,6 @@ whereptr: NonNull<ggml_sys::ggml_tensor>,}-impl Tensor<1> {- pub fn copy_from_slice(&mut self, slice: &[f32]) {- if slice.len() != self.shape()[0] as usize {- panic!("slice length does not match tensor shape");- }-- if self.type_ != GgmlType::F32 {- panic!("tensor type does not match slice type");- }-- unsafe {- let ptr = self.ptr.as_ptr().as_ref().unwrap().data;- (ptr as *mut f32).copy_from_nonoverlapping(slice.as_ptr(), slice.len());- }- }-- pub fn copy_to_slice(&self, out: &mut [f32]) {- if out.len() != self.shape()[0] as usize {- panic!("slice length does not match tensor shape");- }-- if self.type_ != GgmlType::F32 {- panic!("tensor type does not match slice type");- }-- let data = unsafe {- std::slice::from_raw_parts(- self.ptr.as_ptr().as_ref().unwrap().data as *const f32,- self.shape()[0] as usize,- )- };-- out.copy_from_slice(data);- }-}-impl<const DIMS: usize> Tensor<DIMS>whereDims<DIMS>: ValidDims,@@ -155,6 +120,48 @@ whereself.ptr.as_ptr()}+ pub fn num_bytes(&self) -> usize {+ unsafe { ggml_sys::ggml_nbytes(self.ptr.as_ptr()) }+ }++ pub fn copy_from_slice(&mut self, slice: &[f32]) {+ if slice.len() as u64 != self.num_elements() {+ panic!("slice length does not match tensor shape");+ }++ if self.type_ != GgmlType::F32 {+ panic!("tensor type does not match slice type");+ }++ unsafe {+ let ptr = self.ptr.as_ptr().as_ref().unwrap().data;+ (ptr as *mut f32).copy_from_nonoverlapping(slice.as_ptr(), slice.len());+ }+ }++ pub fn copy_to_slice(&self, out: &mut [f32]) {+ if out.len() as u64 != self.num_elements() {+ panic!("slice length does not match tensor shape");+ }++ if self.type_ != GgmlType::F32 {+ panic!("tensor type does not match slice type");+ }++ let data = unsafe {+ std::slice::from_raw_parts(+ self.ptr.as_ptr().as_ref().unwrap().data as *const f32,+ self.num_elements() as usize,+ )+ };++ out.copy_from_slice(data);+ }++ pub fn num_elements(&self) -> u64 {+ self.shape().into_iter().product()+ }+pub fn shape(&self) -> [u64; DIMS] {let shape = unsafe { self.ptr.as_ptr().as_ref().unwrap().ne };@@ -175,7 +182,7 @@ whereDims<ROW_DIMS>: ValidDims,Dims<OUT_DIMS>: DimsPlusOne<ROW_DIMS> + ValidDims,{- assert_eq!(rows.type_, GgmlType::I32, "rows tensor must be of type I32");+ debug_assert_eq!(rows.type_, GgmlType::I32, "rows tensor must be of type I32");let ptr = unsafe {ggml_sys::ggml_get_rows(self.ctx.as_ptr(), self.ptr.as_ptr(), rows.ptr.as_ptr())@@ -209,7 +216,7 @@ where{debug_assert_eq!(shape.into_iter().product::<u64>(),- self.shape().into_iter().product::<u64>(),+ self.num_elements(),"reshape must not change the number of elements");@@ -338,6 +345,46 @@ whereptr: NonNull::new(ptr).unwrap(),}}++ pub fn as_type(&self, type_: GgmlType) -> Tensor<DIMS> {+ let ptr =+ unsafe { ggml_sys::ggml_cast(self.ctx.as_ptr(), self.ptr.as_ptr(), type_.as_raw()) };++ Tensor {+ ctx: Arc::clone(&self.ctx),+ type_,+ ptr: NonNull::new(ptr).unwrap(),+ }+ }++ pub fn norm(&self, eps: f32) -> Tensor<DIMS> {+ let ptr = unsafe { ggml_sys::ggml_norm(self.ctx.as_ptr(), self.ptr.as_ptr(), eps) };++ Tensor {+ ctx: Arc::clone(&self.ctx),+ type_: self.type_,+ ptr: NonNull::new(ptr).unwrap(),+ }+ }++ pub fn load_bytes(&mut self, bytes: &[u8]) {+ debug_assert_eq!(+ bytes.len(),+ self.num_bytes(),+ "bytes must be the same size as the tensor"+ );++ unsafe {+ debug_assert_ne!(+ self.as_ptr().as_mut().unwrap().data,+ std::ptr::null_mut(),+ "tensor must be allocated"+ );++ (self.as_ptr().as_mut().unwrap().data as *mut u8)+ .copy_from_nonoverlapping(bytes.as_ptr(), self.num_bytes());+ }+ }}macro_rules! std_ops_impl {@@ -364,6 +411,50 @@ macro_rules! std_ops_impl {}}+ impl<'a, const DIMS: usize> std::ops::$trait<Tensor<DIMS>> for &'a Tensor<DIMS>+ where+ Dims<DIMS>: ValidDims,+ {+ type Output = Tensor<DIMS>;+ fn $std_func(self, rhs: Tensor<DIMS>) -> Self::Output {+ if self.type_ != rhs.type_ {+ panic!("tensor types do not match");+ }++ let ptr = unsafe {+ ggml_sys::$func(self.ctx.as_ptr(), self.ptr.as_ptr(), rhs.ptr.as_ptr())+ };++ Self::Output {+ ctx: Arc::clone(&self.ctx),+ type_: self.type_,+ ptr: NonNull::new(ptr).unwrap(),+ }+ }+ }++ impl<'b, const DIMS: usize> std::ops::$trait<&'b Tensor<DIMS>> for Tensor<DIMS>+ where+ Dims<DIMS>: ValidDims,+ {+ type Output = Tensor<DIMS>;+ fn $std_func(self, rhs: &'b Tensor<DIMS>) -> Self::Output {+ if self.type_ != rhs.type_ {+ panic!("tensor types do not match");+ }++ let ptr = unsafe {+ ggml_sys::$func(self.ctx.as_ptr(), self.ptr.as_ptr(), rhs.ptr.as_ptr())+ };++ Self::Output {+ ctx: Arc::clone(&self.ctx),+ type_: self.type_,+ ptr: NonNull::new(ptr).unwrap(),+ }+ }+ }+impl<'a, 'b, const DIMS: usize> std::ops::$trait<&'b Tensor<DIMS>> for &'a Tensor<DIMS>whereDims<DIMS>: ValidDims,@@ -392,21 +483,6 @@ std_ops_impl!(Add, add, ggml_add);std_ops_impl!(Sub, sub, ggml_sub);std_ops_impl!(Mul, mul, ggml_mul_mat);-// impl<const DIMS: usize> Tensor<DIMS> {-// pub fn concat(&self, other: &Tensor<DIMS>) -> Tensor<DIMS+1> {-// let ptr = unsafe {-// ggml_sys::ggml_concat(self.ctx.as_ptr(), self.ptr.as_ptr(), other.ptr.as_ptr())-// };-//-// Self {-// ctx: Arc::clone(&self.ctx),-// type_: self.type_,-// shape: self.shape,-// ptr: NonNull::new(ptr).unwrap(),-// }-// }-// }-#[cfg(test)]mod tests {use super::*;@@ -442,4 +518,18 @@ mod tests {test_1d!(add_1d, add, [1.0, 2.0], [2.0, 2.0], [3.0, 4.0]);test_1d!(mul_1d, mul, [2.0, 5.0], [2.0, 3.0], [19.0]);test_1d!(sub_1d, sub, [3.0, 10.0], [2.0, 4.0], [1.0, 6.0]);++ #[test]+ fn num_bytes() {+ let mut ctx = Context::new(128 * 1024 * 1024, 1);+ let at = Tensor::new(&mut ctx, GgmlType::F32, [2, 3, 4]);+ assert_eq!(at.num_bytes(), 32 * 2 * 3 * 4 / 8);+ }++ #[test]+ fn load_bytes() {+ let mut ctx = Context::new(128 * 1024 * 1024, 1);+ let mut at = Tensor::new(&mut ctx, GgmlType::F32, [2]);+ at.load_bytes(&[1, 2, 3, 4, 5, 6, 7, 8]);+ }}