Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Tokenizer owned by backend (#730)

uuuvn9ce3d522cfd5
diff --git a/crates/backend-uzu/src/bridge/backend.rs b/crates/backend-uzu/src/bridge/backend.rs
index ce0da6bee..607e3c361 100644
--- a/crates/backend-uzu/src/bridge/backend.rs
+++ b/crates/backend-uzu/src/bridge/backend.rs
@@ -1,4 +1,4 @@
-use std::{pin::Pin, sync::Arc};
+use std::pin::Pin;
use shoji::{
traits::{
@@ -11,7 +11,6 @@ use shoji::{
},
types::session::chat::ChatConfig,
};
-use tokenizers::Tokenizer;
use crate::{
TOOLCHAIN_VERSION,
@@ -52,17 +51,11 @@ impl ChatTokenBackend for UzuLlmBackend {
&'a self,
reference: String,
config: ChatConfig,
- #[cfg_attr(not(grammar), allow(unused_variables))] tokenizer: Arc<Tokenizer>,
) -> Pin<Box<dyn Future<Output = Result<Box<dyn ChatTokenInstance>, BackendError>> + Send + 'a>> {
Box::pin(async move {
let instance = select_backend!(
- UzuChatTokenBackendInstance::<B>::new(
- reference,
- config,
- #[cfg(grammar)]
- tokenizer,
- )
- .map(|i| Box::new(i) as Box<dyn ChatTokenInstance>),
+ UzuChatTokenBackendInstance::<B>::new(reference, config)
+ .map(|i| Box::new(i) as Box<dyn ChatTokenInstance>),
BackendError::from("Unable to open any backend")
)?;
Ok(instance)
diff --git a/crates/backend-uzu/src/bridge/chat_token_backend.rs b/crates/backend-uzu/src/bridge/chat_token_backend.rs
index 01fa5dc95..328ad404d 100644
--- a/crates/backend-uzu/src/bridge/chat_token_backend.rs
+++ b/crates/backend-uzu/src/bridge/chat_token_backend.rs
@@ -21,7 +21,6 @@ use shoji::{
},
types::session::chat::{ChatConfig, ChatReplyConfig},
};
-#[cfg(grammar)]
use tokenizers::Tokenizer;
use tokio_util::sync::CancellationToken;
@@ -47,8 +46,6 @@ pub struct UzuChatTokenBackendInstance<B: Backend> {
engine: Arc<Engine<B>>,
model: Arc<LanguageModel<B>>,
config: ChatConfig,
- #[cfg(grammar)]
- tokenizer: Arc<Tokenizer>,
stop_token_ids: Vec<i32>,
max_context_length: Option<u32>,
}
@@ -57,7 +54,6 @@ impl<B: Backend> UzuChatTokenBackendInstance<B> {
pub fn new(
model_path: String,
config: ChatConfig,
- #[cfg(grammar)] tokenizer: Arc<Tokenizer>,
) -> Result<Self, BackendError> {
let engine = Engine::<B>::new().map_err(|err| err.to_string())?;
let model_path = PathBuf::from(model_path);
@@ -70,8 +66,6 @@ impl<B: Backend> UzuChatTokenBackendInstance<B> {
engine,
model: Arc::new(model),
config,
- #[cfg(grammar)]
- tokenizer,
stop_token_ids,
max_context_length,
})
@@ -117,7 +111,7 @@ impl<B: Backend> BackendInstance for UzuChatTokenBackendInstance<B> {
#[cfg(grammar)]
let grammar = if let Some(grammar_config) = config.grammar {
- match get_grammar(grammar_config, self.tokenizer.as_ref(), &self.stop_token_ids) {
+ match get_grammar(grammar_config, self.model.tokenizer(), &self.stop_token_ids) {
Ok(grammar) => Some(grammar),
Err(err) => {
return Box::pin(NoMetricsStream::new(error_stream(err.to_string())));
@@ -166,6 +160,10 @@ impl<B: Backend> BackendInstance for UzuChatTokenBackendInstance<B> {
}
impl<B: Backend> ChatTokenBackendInstance for UzuChatTokenBackendInstance<B> {
+ fn tokenizer(&self) -> Arc<Tokenizer> {
+ self.model.tokenizer().clone()
+ }
+
fn max_context_length(&self) -> Option<usize> {
self.max_context_length.map(|max_context_length| max_context_length as usize)
}
diff --git a/crates/backend-uzu/src/bridge/classification_backend.rs b/crates/backend-uzu/src/bridge/classification_backend.rs
index c516bd694..b837caefb 100644
--- a/crates/backend-uzu/src/bridge/classification_backend.rs
+++ b/crates/backend-uzu/src/bridge/classification_backend.rs
@@ -2,20 +2,26 @@ use std::{path::PathBuf, pin::Pin, sync::Arc};
use futures::{StreamExt, stream};
use parking_lot::Mutex;
-use shoji::traits::{
- State as ShojiState,
- backend::{
- Error as BackendError, Instance as BackendInstance, InstanceStream, NoMetricsStream,
- classification::{
- StreamConfig as ClassificationStreamConfig, StreamInput as ClassificationStreamInput,
- StreamMetrics as ClassificationStreamMetrics, StreamOutput as ClassificationStreamOutput,
+use shoji::{
+ traits::{
+ State as ShojiState,
+ backend::{
+ Error as BackendError, Instance as BackendInstance, InstanceStream, NoMetricsStream,
+ classification::{
+ Instance as ClassificationInstance, StreamConfig as ClassificationStreamConfig,
+ StreamInput as ClassificationStreamInput, StreamMetrics as ClassificationStreamMetrics,
+ StreamOutput as ClassificationStreamOutput,
+ },
},
},
+ types::session::classification::{ChatTokenCodecConfig, TokenCodecConfig},
};
+use tokenizers::Tokenizer;
use tokio_util::sync::CancellationToken;
use crate::{
backends::common::Backend,
+ config::token_codec::AnyTokenCodecConfig,
engine::{Engine, classifier_model::ClassifierModel},
};
@@ -71,6 +77,29 @@ impl<B: Backend> BackendInstance for UzuClassificationBackendInstance<B> {
}
}
+impl<B: Backend> ClassificationInstance for UzuClassificationBackendInstance<B> {
+ fn tokenizer(&self) -> Arc<Tokenizer> {
+ self.model.lock().tokenizer().clone()
+ }
+
+ fn token_codec_config(&self) -> TokenCodecConfig {
+ match self.model.lock().token_codec_config() {
+ AnyTokenCodecConfig::ChatCodecConfig(config) => TokenCodecConfig::Chat(ChatTokenCodecConfig {
+ prompt_template: config.prompt_template.clone(),
+ output_parser_regex: config.output_parser_regex.clone(),
+ system_role_name: config.system_role_name.clone(),
+ user_role_name: config.user_role_name.clone(),
+ assistant_role_name: config.assistant_role_name.clone(),
+ eos_token: config.eos_token.clone(),
+ bos_token: config.bos_token.clone(),
+ end_of_thinking_tag: config.end_of_thinking_tag.clone(),
+ default_system_prompt: config.default_system_prompt.clone(),
+ }),
+ AnyTokenCodecConfig::RawTextCodecConfig(_) => TokenCodecConfig::RawText,
+ }
+ }
+}
+
#[derive(Debug, Clone)]
struct State;
diff --git a/crates/backend-uzu/src/engine/classifier_model.rs b/crates/backend-uzu/src/engine/classifier_model.rs
index 324c8ecc5..c89653f88 100644
--- a/crates/backend-uzu/src/engine/classifier_model.rs
+++ b/crates/backend-uzu/src/engine/classifier_model.rs
@@ -10,10 +10,11 @@ use std::{
use half::bf16;
use shoji::traits::backend::classification::ClassifierOutput;
use thiserror::Error;
+use tokenizers::Tokenizer;
use crate::{
backends::common::{AllocationType, Backend, Context, Encoder},
- config::model::classifier_model::ClassifierModelConfig,
+ config::{model::classifier_model::ClassifierModelConfig, token_codec::AnyTokenCodecConfig},
data_type::DataType,
encodable_block::classifier::{Classifier as ClassifierEncodable, ClassifierError as ClassifierEncodableError},
engine::Engine,
@@ -25,6 +26,8 @@ pub struct ClassifierModel<B: Backend> {
classifier: ClassifierEncodable<B>,
output_labels: Box<[String]>,
data_type: DataType,
+ tokenizer: Arc<Tokenizer>,
+ token_codec_config: AnyTokenCodecConfig,
}
#[derive(Debug, Error)]
@@ -39,6 +42,8 @@ pub enum EngineLoadClassifierError<B: Backend> {
ParameterLoader(#[from] ParameterLoaderError<B>),
#[error("Classifier error: {0}")]
Classifier(#[from] ClassifierEncodableError<B>),
+ #[error("Tokenizer error: {0}")]
+ Tokenizer(#[from] tokenizers::Error),
}
impl<B: Backend> Engine<B> {
@@ -72,11 +77,16 @@ impl<B: Backend> Engine<B> {
weight_loader.tree().assert_all_tensors_validated()?;
+ let tokenizer = Arc::new(Tokenizer::from_file(model_path.join("tokenizer.json"))?);
+ let token_codec_config = config.token_codec_config;
+
Ok(ClassifierModel {
context,
classifier,
output_labels,
data_type,
+ tokenizer,
+ token_codec_config,
})
}
}
@@ -94,6 +104,14 @@ pub enum ClassifierModelClassifyError<B: Backend> {
}
impl<B: Backend> ClassifierModel<B> {
+ pub fn tokenizer(&self) -> &Arc<Tokenizer> {
+ &self.tokenizer
+ }
+
+ pub fn token_codec_config(&self) -> &AnyTokenCodecConfig {
+ &self.token_codec_config
+ }
+
pub fn classify(
&self,
input: &[u64],
diff --git a/crates/backend-uzu/src/engine/language_model/mod.rs b/crates/backend-uzu/src/engine/language_model/mod.rs
index 6148e2379..94c5d315f 100644
--- a/crates/backend-uzu/src/engine/language_model/mod.rs
+++ b/crates/backend-uzu/src/engine/language_model/mod.rs
@@ -1,6 +1,7 @@
use std::{fs::File, io, io::BufReader, path::Path, sync::Arc};
use thiserror::Error;
+use tokenizers::Tokenizer;
use crate::{
backends::common::{Backend, Context, DeviceCapabilities, Kernels, kernel::ContextRingUpdateKernel},
@@ -28,6 +29,7 @@ pub struct LanguageModel<B: Backend> {
sampling: Sampling<B>,
context_ring_update: <B::Kernels as Kernels>::ContextRingUpdateKernel,
generation_config: GenerationConfig,
+ tokenizer: Arc<Tokenizer>,
#[cfg(grammar)]
vocab_size: usize,
}
@@ -48,6 +50,8 @@ pub enum EngineLoadLanguageModelError<B: Backend> {
Decoder(#[from] DecoderError<B>),
#[error("Speculator error: {0}")]
Speculator(#[from] DFlashSpeculatorLoadError<B>),
+ #[error("Tokenizer error: {0}")]
+ Tokenizer(#[from] tokenizers::Error),
}
impl<B: Backend> Engine<B> {
@@ -65,30 +69,8 @@ impl<B: Backend> Engine<B> {
let speculator_path = model_path.join("speculator");
let speculator_path = speculator_path.exists().then_some(speculator_path);
- self.build_language_model(config, &weight_loader, speculator_path.as_deref())
- }
-
- // TODO: better design
- pub fn load_language_model_random(
- self: &Arc<Self>,
- config_path: &Path,
- header_path: &Path,
- seed: u64,
- ) -> Result<LanguageModel<B>, EngineLoadLanguageModelError<B>> {
- let config: LanguageModelConfig = serde_json::from_reader(BufReader::new(File::open(config_path)?))?;
-
- let header_file = File::open(header_path)?;
- let weight_loader = ParameterLoader::new_random(&header_file, &*self.context, seed)?;
+ let tokenizer = Arc::new(Tokenizer::from_file(model_path.join("tokenizer.json"))?);
- self.build_language_model(config, &weight_loader, None)
- }
-
- fn build_language_model(
- self: &Arc<Self>,
- config: LanguageModelConfig,
- weight_loader: &ParameterLoader<B>,
- speculator_path: Option<&Path>,
- ) -> Result<LanguageModel<B>, EngineLoadLanguageModelError<B>> {
let data_type = DataType::BF16;
let decoder = Decoder::new(
@@ -104,6 +86,7 @@ impl<B: Backend> Engine<B> {
);
let speculator = speculator_path
+ .as_deref()
.map(|speculator_path| DFlashTfmSpeculator::new(speculator_path, self.context.clone()))
.transpose()?;
@@ -126,6 +109,7 @@ impl<B: Backend> Engine<B> {
sampling,
context_ring_update,
generation_config,
+ tokenizer,
#[cfg(grammar)]
vocab_size,
})
@@ -182,4 +166,8 @@ impl<B: Backend> LanguageModel<B> {
pub fn generation_config(&self) -> &GenerationConfig {
&self.generation_config
}
+
+ pub fn tokenizer(&self) -> &Arc<Tokenizer> {
+ &self.tokenizer
+ }
}
diff --git a/crates/backend-uzu/src/parameters/loader.rs b/crates/backend-uzu/src/parameters/loader.rs
index 31e7e564e..807c4ee1e 100644
--- a/crates/backend-uzu/src/parameters/loader.rs
+++ b/crates/backend-uzu/src/parameters/loader.rs
@@ -4,8 +4,6 @@ use std::{
fs::File,
};
-use half::{bf16, f16};
-use rand::{RngExt, SeedableRng, rngs::SmallRng};
use thiserror::Error;
use super::safetensors_metadata::{HeaderLoadingError, read_metadata as read_st_metadata};
@@ -54,17 +52,12 @@ pub enum ParameterLoaderError<B: Backend> {
},
}
-enum ParameterBytes<'a> {
- File(&'a File),
- Random(u64),
-}
-
pub struct ParameterLoader<'a, B: Backend> {
context: &'a B::Context,
index: HashMap<String, ParameterMetadata>,
metadata: HashMap<String, String>,
validated_tensors: RefCell<HashSet<String>>,
- bytes: ParameterBytes<'a>,
+ file: &'a File,
}
impl<'a, B: Backend> ParameterLoader<'a, B> {
@@ -72,23 +65,7 @@ impl<'a, B: Backend> ParameterLoader<'a, B> {
file: &'a File,
context: &'a B::Context,
) -> Result<Self, HeaderLoadingError> {
- Self::from_header(file, context, ParameterBytes::File(file))
- }
-
- pub fn new_random(
- header_file: &File,
- context: &'a B::Context,
- seed: u64,
- ) -> Result<Self, HeaderLoadingError> {
- Self::from_header(header_file, context, ParameterBytes::Random(seed))
- }
-
- fn from_header(
- header_file: &File,
- context: &'a B::Context,
- bytes: ParameterBytes<'a>,
- ) -> Result<Self, HeaderLoadingError> {
- let (global_offset, st_metadata) = read_st_metadata(header_file)?;
+ let (global_offset, st_metadata) = read_st_metadata(file)?;
let index = st_metadata
.tensors
.into_iter()
@@ -121,7 +98,7 @@ impl<'a, B: Backend> ParameterLoader<'a, B> {
index,
metadata,
validated_tensors: RefCell::new(HashSet::new()),
- bytes,
+ file,
})
}
@@ -178,12 +155,7 @@ impl<'a, 'leaf, B: Backend> ParameterLeaf<'a, 'leaf, B, true> {
let element_count = self.metadata.size / std::mem::size_of::<T>();
let mut data = vec![T::zeroed(); element_count];
let destination = bytemuck::cast_slice_mut(&mut data);
- match &self.loader.bytes {
- ParameterBytes::File(file) => {
- file_read_exact_at(file, destination, self.metadata.offset as u64)?;
- },
- ParameterBytes::Random(seed) => fill_random(destination, self.metadata.data_type, *seed),
- }
+ file_read_exact_at(self.loader.file, destination, self.metadata.offset as u64)?;
Ok(data.into_boxed_slice())
}
@@ -201,60 +173,11 @@ impl<'a, 'leaf, B: Backend> ParameterLeaf<'a, 'leaf, B, true> {
range.len(),
)
};
- match &self.loader.bytes {
- ParameterBytes::File(file) => {
- file_read_exact_at(file, destination, self.metadata.offset as u64)?;
- },
- ParameterBytes::Random(seed) => fill_random(destination, self.metadata.data_type, *seed),
- }
+ file_read_exact_at(self.loader.file, destination, self.metadata.offset as u64)?;
Ok(allocation)
}
}
-fn fill_random(
- destination: &mut [u8],
- data_type: DataType,
- seed: u64,
-) {
- if destination.is_empty() {
- return;
- }
- let mut rng = SmallRng::seed_from_u64(seed);
- let head_len = destination.len().min(65536);
- let (head, tail) = destination.split_at_mut(head_len);
- match data_type {
- DataType::BF16 => {
- for chunk in head.as_chunks_mut::<2>().0 {
- *chunk = bf16::from_f32(rng.random_range(-0.1f32..0.1f32)).to_le_bytes();
- }
- },
- DataType::F16 => {
- for chunk in head.as_chunks_mut::<2>().0 {
- *chunk = f16::from_f32(rng.random_range(-0.1f32..0.1f32)).to_le_bytes();
- }
- },
- DataType::F32 => {
- for chunk in head.as_chunks_mut::<4>().0 {
- *chunk = rng.random_range(-0.1f32..0.1f32).to_le_bytes();
- }
- },
- DataType::F64 => {
- for chunk in head.as_chunks_mut::<8>().0 {
- *chunk = rng.random_range(-0.1f64..0.1f64).to_le_bytes();
- }
- },
- _ => {
- for chunk in head.chunks_mut(8) {
- let bytes = rng.random::<u64>().to_le_bytes();
- chunk.copy_from_slice(&bytes[..chunk.len()]);
- }
- },
- }
- for target in tail.chunks_mut(head_len) {
- target.copy_from_slice(&head[..target.len()]);
- }
-}
-
pub struct ParameterTree<'loader, B: Backend> {
loader: &'loader ParameterLoader<'loader, B>,
prefix: Option<String>,
diff --git a/crates/hanashi/src/chat/hanashi/error.rs b/crates/hanashi/src/chat/hanashi/error.rs
index 839b5b7ba..9d6bbdc8b 100644
--- a/crates/hanashi/src/chat/hanashi/error.rs
+++ b/crates/hanashi/src/chat/hanashi/error.rs
@@ -12,8 +12,6 @@ pub enum Error {
ConfigNotFound(String),
#[error("Config '{0}' is invalid")]
InvalidConfig(String),
- #[error("Unable to load tokenizer")]
- UnableToLoadTokenizer,
#[error("Unable to encode text")]
UnableToEncodeText,
#[error("Unable to decode token")]
diff --git a/crates/hanashi/src/classification/config.rs b/crates/hanashi/src/classification/config.rs
deleted file mode 100644
index 059ff78cb..000000000
--- a/crates/hanashi/src/classification/config.rs
+++ /dev/null
@@ -1,39 +0,0 @@
-use std::{io, io::ErrorKind};
-
-use serde_json::Value;
-
-pub enum TokenCodecConfig {
- Chat(ChatTokenCodecConfig),
- RawText,
-}
-
-impl TokenCodecConfig {
- pub fn from_value(value: Value) -> Result<Self, io::Error> {
- let typ = value.get("type");
- match typ {
- Some(Value::String(codec_type)) if codec_type == "RawTextCodecConfig" => Ok(Self::RawText),
- Some(Value::String(codec_type)) if codec_type == "ChatCodecConfig" => {
- let config = serde_json::from_value::<ChatTokenCodecConfig>(value)?;
- Ok(Self::Chat(config))
- },
- None => {
- let config = serde_json::from_value::<ChatTokenCodecConfig>(value)?;
- Ok(Self::Chat(config))
- },
- _ => Err(io::Error::new(ErrorKind::InvalidData, format!("Invalid config: unknown type \"{typ:?}\""))),
- }
- }
-}
-
-#[derive(Debug, Clone, PartialEq, ::serde::Serialize, ::serde::Deserialize)]
-pub struct ChatTokenCodecConfig {
- pub prompt_template: String,
- pub output_parser_regex: Option<String>,
- pub system_role_name: String,
- pub user_role_name: String,
- pub assistant_role_name: String,
- pub eos_token: Option<String>,
- pub bos_token: Option<String>,
- pub end_of_thinking_tag: Option<String>,
- pub default_system_prompt: Option<String>,
-}
diff --git a/crates/hanashi/src/classification/encoding.rs b/crates/hanashi/src/classification/encoding.rs
index 8060080fc..5edfbda33 100644
--- a/crates/hanashi/src/classification/encoding.rs
+++ b/crates/hanashi/src/classification/encoding.rs
@@ -1,56 +1,37 @@
-use std::{collections::HashMap, fs::File, path::PathBuf, string::ToString};
+use std::{collections::HashMap, string::ToString, sync::Arc};
use minijinja::{Environment, context};
use minijinja_contrib::pycompat::unknown_method_callback;
-use serde_json::Value;
-use shoji::types::session::classification::{ClassificationMessage, ClassificationRole};
+use shoji::types::session::classification::{
+ ChatTokenCodecConfig, ClassificationMessage, ClassificationRole, TokenCodecConfig,
+};
use tokenizers::Tokenizer;
-use crate::{
- chat::{
- TokenizerLocation,
- hanashi::{
- Error,
- renderer::{
- RAISE_EXCEPTION_FUNCTION_NAME, STRFTIME_NOW_FUNCTION_NAME, TEMPLATE_NAME, TOJSON_FILTER_NAME,
- raise_exception, to_json,
- },
+use crate::chat::{
+ hanashi::{
+ Error,
+ renderer::{
+ RAISE_EXCEPTION_FUNCTION_NAME, STRFTIME_NOW_FUNCTION_NAME, TEMPLATE_NAME, TOJSON_FILTER_NAME,
+ raise_exception, to_json,
},
- strftime_now,
},
- classification::config::{ChatTokenCodecConfig, TokenCodecConfig},
- util::tokenizer::load_tokenizer,
+ strftime_now,
};
-const CONFIG_FILE_NAME: &str = "config.json";
-
pub struct ClassificationEncoding {
- tokenizer: Tokenizer,
+ tokenizer: Arc<Tokenizer>,
config: TokenCodecConfig,
}
impl ClassificationEncoding {
- pub fn new(model_dir: &str) -> Result<Self, Error> {
- let model_path = PathBuf::from(model_dir);
- let config_file = File::open(model_path.join(CONFIG_FILE_NAME))
- .map_err(|_| Error::ConfigNotFound(CONFIG_FILE_NAME.to_string()))?;
- let config = serde_json::from_reader::<File, Value>(config_file)
- .map_err(|_| Error::InvalidConfig(CONFIG_FILE_NAME.to_string()))?;
- let codec_config =
- config.get("token_codec_config").ok_or_else(|| Error::InvalidConfig(CONFIG_FILE_NAME.to_string()))?;
- let config = TokenCodecConfig::from_value(codec_config.clone())
- .map_err(|_| Error::InvalidConfig(CONFIG_FILE_NAME.to_string()))?;
-
- let tokenizer_location = TokenizerLocation::Directory {
- path: model_dir.to_string(),
- name: None,
- };
- let tokenizer = load_tokenizer(&tokenizer_location).map_err(|_| Error::UnableToLoadTokenizer)?;
-
- Ok(Self {
+ pub fn new(
+ config: TokenCodecConfig,
+ tokenizer: Arc<Tokenizer>,
+ ) -> Self {
+ Self {
tokenizer,
config,
- })
+ }
}
pub fn encode(
diff --git a/crates/hanashi/src/classification/mod.rs b/crates/hanashi/src/classification/mod.rs
index 6f2940021..295694271 100644
--- a/crates/hanashi/src/classification/mod.rs
+++ b/crates/hanashi/src/classification/mod.rs
@@ -1,2 +1 @@
-mod config;
pub mod encoding;
diff --git a/crates/hanashi/src/lib.rs b/crates/hanashi/src/lib.rs
index ecca3db9c..d767d6176 100644
--- a/crates/hanashi/src/lib.rs
+++ b/crates/hanashi/src/lib.rs
@@ -1,10 +1,8 @@
mod classification;
-mod util;
pub mod chat;
pub use classification::encoding::ClassificationEncoding;
-pub use util::tokenizer::load_tokenizer;
pub trait Encoding {
type Config;
diff --git a/crates/hanashi/src/util/mod.rs b/crates/hanashi/src/util/mod.rs
deleted file mode 100644
index d9b8fd121..000000000
--- a/crates/hanashi/src/util/mod.rs
+++ /dev/null
@@ -1 +0,0 @@
-pub mod tokenizer;
diff --git a/crates/hanashi/src/util/tokenizer.rs b/crates/hanashi/src/util/tokenizer.rs
deleted file mode 100644
index 522fc6d00..000000000
--- a/crates/hanashi/src/util/tokenizer.rs
+++ /dev/null
@@ -1,23 +0,0 @@
-use std::path::PathBuf;
-
-use tokenizers::Tokenizer;
-
-use crate::chat::TokenizerLocation;
-
-const DEFAULT_TOKENIZER_FILENAME: &str = "tokenizer.json";
-
-pub fn load_tokenizer(location: &TokenizerLocation) -> Result<Tokenizer, Box<dyn std::error::Error + Send + Sync>> {
- let tokenizer_path = match location {
- TokenizerLocation::File {
- path,
- } => PathBuf::from(path),
- TokenizerLocation::Directory {
- path,
- name,
- } => match name {
- Some(name) => PathBuf::from(path).join(name),
- None => PathBuf::from(path).join(DEFAULT_TOKENIZER_FILENAME),
- },
- };
- Tokenizer::from_file(&tokenizer_path)
-}
diff --git a/crates/nagare/src/chat/token.rs b/crates/nagare/src/chat/token.rs
index 33a59e7ee..569571712 100644
--- a/crates/nagare/src/chat/token.rs
+++ b/crates/nagare/src/chat/token.rs
@@ -1,7 +1,6 @@
use std::{
io,
pin::Pin,
- sync::Arc,
time::{Duration, Instant},
};
@@ -9,7 +8,6 @@ use futures::{Stream, StreamExt, stream};
use hanashi::{
Encoding as EncodingTrait,
chat::{Encoding, EncodingConfig, TokenizerLocation, hanashi::HanashiEncodingImpl, harmony::HarmonyEncodingImpl},
- load_tokenizer,
};
use shoji::{
traits::{
@@ -81,15 +79,16 @@ impl Session {
path: reference.clone(),
name: None,
};
- let tokenizer = load_tokenizer(&tokenizer_location).map_err(|err| ChatSessionError::Loading {
- message: format!("Failed to initialize tokenizer: {err}"),
- })?;
- let tokenizer = Arc::new(tokenizer);
+
+ let instance =
+ backend.instance(reference.clone(), config).await.map_err(|error| ChatSessionError::Backend {
+ message: error.to_string(),
+ })?;
let encoding = match encoding_config {
Some(EncodingConfig::Hanashi {
config,
- }) => HanashiEncodingImpl::new(config, tokenizer.clone()).map(Encoding::Hanashi).map_err(|err| {
+ }) => HanashiEncodingImpl::new(config, instance.tokenizer()).map(Encoding::Hanashi).map_err(|err| {
ChatSessionError::Loading {
message: format!("can not create harmony encoding: {err}"),
}
@@ -105,12 +104,6 @@ impl Session {
message: "can not get encoding config".to_string(),
}),
}?;
-
- let instance = backend.instance(reference.clone(), config, tokenizer).await.map_err(|error| {
- ChatSessionError::Backend {
- message: error.to_string(),
- }
- })?;
let state = instance.state().await.map_err(|error| ChatSessionError::Backend {
message: error.to_string(),
})?;
diff --git a/crates/nagare/src/classification/mod.rs b/crates/nagare/src/classification/mod.rs
index 312a1ef2f..7c159a3fa 100644
--- a/crates/nagare/src/classification/mod.rs
+++ b/crates/nagare/src/classification/mod.rs
@@ -63,10 +63,7 @@ impl ClassificationSession {
let instance_state = instance.state().await.map_err(|error| ClassificationSessionError::Backend {
message: error.to_string(),
})?;
- let encoding =
- ClassificationEncoding::new(reference.as_str()).map_err(|err| ClassificationSessionError::Backend {
- message: err.to_string(),
- })?;
+ let encoding = ClassificationEncoding::new(instance.token_codec_config(), instance.tokenizer());
Ok(InstanceHolder {
instance,
diff --git a/crates/shoji/src/traits/backend/chat_token.rs b/crates/shoji/src/traits/backend/chat_token.rs
index 5aa3ce9cf..f77d28f89 100644
--- a/crates/shoji/src/traits/backend/chat_token.rs
+++ b/crates/shoji/src/traits/backend/chat_token.rs
@@ -31,7 +31,6 @@ pub trait Backend: Send + Sync {
&'a self,
reference: String,
config: ChatConfig,
- tokenizer: Arc<Tokenizer>,
) -> Pin<Box<dyn Future<Output = Result<Box<dyn Instance>, Error>> + Send + 'a>>;
}
@@ -43,6 +42,8 @@ pub trait Instance:
StreamMetrics = StreamMetrics,
>
{
+ fn tokenizer(&self) -> Arc<Tokenizer>;
+
fn max_context_length(&self) -> Option<usize>;
fn stop_token_ids(&self) -> Option<Box<[u64]>>;
diff --git a/crates/shoji/src/traits/backend/classification.rs b/crates/shoji/src/traits/backend/classification.rs
index 8111639db..3f529bba6 100644
--- a/crates/shoji/src/traits/backend/classification.rs
+++ b/crates/shoji/src/traits/backend/classification.rs
@@ -1,6 +1,11 @@
-use std::{collections::HashMap, convert::Infallible, pin::Pin};
+use std::{collections::HashMap, convert::Infallible, pin::Pin, sync::Arc};
-use crate::traits::backend::{Error, Instance as InstanceTrait};
+use tokenizers::Tokenizer;
+
+use crate::{
+ traits::backend::{Error, Instance as InstanceTrait},
+ types::session::classification::TokenCodecConfig,
+};
pub struct ClassifierOutput {
pub logits: Vec<f32>,
@@ -29,14 +34,7 @@ pub trait Instance:
StreamMetrics = StreamMetrics,
>
{
-}
+ fn tokenizer(&self) -> Arc<Tokenizer>;
-impl<T> Instance for T where
- T: InstanceTrait<
- StreamConfig = StreamConfig,
- StreamInput = StreamInput,
- StreamOutput = StreamOutput,
- StreamMetrics = StreamMetrics,
- >
-{
+ fn token_codec_config(&self) -> TokenCodecConfig;
}
diff --git a/crates/shoji/src/types/session/classification/codec.rs b/crates/shoji/src/types/session/classification/codec.rs
new file mode 100644
index 000000000..15ba61993
--- /dev/null
+++ b/crates/shoji/src/types/session/classification/codec.rs
@@ -0,0 +1,20 @@
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub enum TokenCodecConfig {
+ Chat(ChatTokenCodecConfig),
+ RawText,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct ChatTokenCodecConfig {
+ pub prompt_template: String,
+ pub output_parser_regex: Option<String>,
+ pub system_role_name: String,
+ pub user_role_name: String,
+ pub assistant_role_name: String,
+ pub eos_token: Option<String>,
+ pub bos_token: Option<String>,
+ pub end_of_thinking_tag: Option<String>,
+ pub default_system_prompt: Option<String>,
+}
diff --git a/crates/shoji/src/types/session/classification/mod.rs b/crates/shoji/src/types/session/classification/mod.rs
index 1b63fb030..6c9c559e3 100644
--- a/crates/shoji/src/types/session/classification/mod.rs
+++ b/crates/shoji/src/types/session/classification/mod.rs
@@ -1,9 +1,11 @@
+mod codec;
mod label;
mod message;
mod output;
mod role;
mod stats;
+pub use codec::{ChatTokenCodecConfig, TokenCodecConfig};
pub use label::ClassificationLabel;
pub use message::ClassificationMessage;
pub use output::{ClassificationOutput, ClassificationOutputProbabilities};