A high-performance inference engine for AI models
Tokenizer owned by backend (#730)
uuuvn9ce3d522cfd5
diff --git a/crates/backend-uzu/src/bridge/backend.rs b/crates/backend-uzu/src/bridge/backend.rsindex ce0da6bee..607e3c361 100644--- a/crates/backend-uzu/src/bridge/backend.rs+++ b/crates/backend-uzu/src/bridge/backend.rs@@ -1,4 +1,4 @@-use std::{pin::Pin, sync::Arc};+use std::pin::Pin;use shoji::{traits::{@@ -11,7 +11,6 @@ use shoji::{},types::session::chat::ChatConfig,};-use tokenizers::Tokenizer;use crate::{TOOLCHAIN_VERSION,@@ -52,17 +51,11 @@ impl ChatTokenBackend for UzuLlmBackend {&'a self,reference: String,config: ChatConfig,- #[cfg_attr(not(grammar), allow(unused_variables))] tokenizer: Arc<Tokenizer>,) -> Pin<Box<dyn Future<Output = Result<Box<dyn ChatTokenInstance>, BackendError>> + Send + 'a>> {Box::pin(async move {let instance = select_backend!(- UzuChatTokenBackendInstance::<B>::new(- reference,- config,- #[cfg(grammar)]- tokenizer,- )- .map(|i| Box::new(i) as Box<dyn ChatTokenInstance>),+ UzuChatTokenBackendInstance::<B>::new(reference, config)+ .map(|i| Box::new(i) as Box<dyn ChatTokenInstance>),BackendError::from("Unable to open any backend"))?;Ok(instance)diff --git a/crates/backend-uzu/src/bridge/chat_token_backend.rs b/crates/backend-uzu/src/bridge/chat_token_backend.rsindex 01fa5dc95..328ad404d 100644--- a/crates/backend-uzu/src/bridge/chat_token_backend.rs+++ b/crates/backend-uzu/src/bridge/chat_token_backend.rs@@ -21,7 +21,6 @@ use shoji::{},types::session::chat::{ChatConfig, ChatReplyConfig},};-#[cfg(grammar)]use tokenizers::Tokenizer;use tokio_util::sync::CancellationToken;@@ -47,8 +46,6 @@ pub struct UzuChatTokenBackendInstance<B: Backend> {engine: Arc<Engine<B>>,model: Arc<LanguageModel<B>>,config: ChatConfig,- #[cfg(grammar)]- tokenizer: Arc<Tokenizer>,stop_token_ids: Vec<i32>,max_context_length: Option<u32>,}@@ -57,7 +54,6 @@ impl<B: Backend> UzuChatTokenBackendInstance<B> {pub fn new(model_path: String,config: ChatConfig,- #[cfg(grammar)] tokenizer: Arc<Tokenizer>,) -> Result<Self, BackendError> {let engine = Engine::<B>::new().map_err(|err| err.to_string())?;let model_path = PathBuf::from(model_path);@@ -70,8 +66,6 @@ impl<B: Backend> UzuChatTokenBackendInstance<B> {engine,model: Arc::new(model),config,- #[cfg(grammar)]- tokenizer,stop_token_ids,max_context_length,})@@ -117,7 +111,7 @@ impl<B: Backend> BackendInstance for UzuChatTokenBackendInstance<B> {#[cfg(grammar)]let grammar = if let Some(grammar_config) = config.grammar {- match get_grammar(grammar_config, self.tokenizer.as_ref(), &self.stop_token_ids) {+ match get_grammar(grammar_config, self.model.tokenizer(), &self.stop_token_ids) {Ok(grammar) => Some(grammar),Err(err) => {return Box::pin(NoMetricsStream::new(error_stream(err.to_string())));@@ -166,6 +160,10 @@ impl<B: Backend> BackendInstance for UzuChatTokenBackendInstance<B> {}impl<B: Backend> ChatTokenBackendInstance for UzuChatTokenBackendInstance<B> {+ fn tokenizer(&self) -> Arc<Tokenizer> {+ self.model.tokenizer().clone()+ }+fn max_context_length(&self) -> Option<usize> {self.max_context_length.map(|max_context_length| max_context_length as usize)}diff --git a/crates/backend-uzu/src/bridge/classification_backend.rs b/crates/backend-uzu/src/bridge/classification_backend.rsindex c516bd694..b837caefb 100644--- a/crates/backend-uzu/src/bridge/classification_backend.rs+++ b/crates/backend-uzu/src/bridge/classification_backend.rs@@ -2,20 +2,26 @@ use std::{path::PathBuf, pin::Pin, sync::Arc};use futures::{StreamExt, stream};use parking_lot::Mutex;-use shoji::traits::{- State as ShojiState,- backend::{- Error as BackendError, Instance as BackendInstance, InstanceStream, NoMetricsStream,- classification::{- StreamConfig as ClassificationStreamConfig, StreamInput as ClassificationStreamInput,- StreamMetrics as ClassificationStreamMetrics, StreamOutput as ClassificationStreamOutput,+use shoji::{+ traits::{+ State as ShojiState,+ backend::{+ Error as BackendError, Instance as BackendInstance, InstanceStream, NoMetricsStream,+ classification::{+ Instance as ClassificationInstance, StreamConfig as ClassificationStreamConfig,+ StreamInput as ClassificationStreamInput, StreamMetrics as ClassificationStreamMetrics,+ StreamOutput as ClassificationStreamOutput,+ },},},+ types::session::classification::{ChatTokenCodecConfig, TokenCodecConfig},};+use tokenizers::Tokenizer;use tokio_util::sync::CancellationToken;use crate::{backends::common::Backend,+ config::token_codec::AnyTokenCodecConfig,engine::{Engine, classifier_model::ClassifierModel},};@@ -71,6 +77,29 @@ impl<B: Backend> BackendInstance for UzuClassificationBackendInstance<B> {}}+impl<B: Backend> ClassificationInstance for UzuClassificationBackendInstance<B> {+ fn tokenizer(&self) -> Arc<Tokenizer> {+ self.model.lock().tokenizer().clone()+ }++ fn token_codec_config(&self) -> TokenCodecConfig {+ match self.model.lock().token_codec_config() {+ AnyTokenCodecConfig::ChatCodecConfig(config) => TokenCodecConfig::Chat(ChatTokenCodecConfig {+ prompt_template: config.prompt_template.clone(),+ output_parser_regex: config.output_parser_regex.clone(),+ system_role_name: config.system_role_name.clone(),+ user_role_name: config.user_role_name.clone(),+ assistant_role_name: config.assistant_role_name.clone(),+ eos_token: config.eos_token.clone(),+ bos_token: config.bos_token.clone(),+ end_of_thinking_tag: config.end_of_thinking_tag.clone(),+ default_system_prompt: config.default_system_prompt.clone(),+ }),+ AnyTokenCodecConfig::RawTextCodecConfig(_) => TokenCodecConfig::RawText,+ }+ }+}+#[derive(Debug, Clone)]struct State;diff --git a/crates/backend-uzu/src/engine/classifier_model.rs b/crates/backend-uzu/src/engine/classifier_model.rsindex 324c8ecc5..c89653f88 100644--- a/crates/backend-uzu/src/engine/classifier_model.rs+++ b/crates/backend-uzu/src/engine/classifier_model.rs@@ -10,10 +10,11 @@ use std::{use half::bf16;use shoji::traits::backend::classification::ClassifierOutput;use thiserror::Error;+use tokenizers::Tokenizer;use crate::{backends::common::{AllocationType, Backend, Context, Encoder},- config::model::classifier_model::ClassifierModelConfig,+ config::{model::classifier_model::ClassifierModelConfig, token_codec::AnyTokenCodecConfig},data_type::DataType,encodable_block::classifier::{Classifier as ClassifierEncodable, ClassifierError as ClassifierEncodableError},engine::Engine,@@ -25,6 +26,8 @@ pub struct ClassifierModel<B: Backend> {classifier: ClassifierEncodable<B>,output_labels: Box<[String]>,data_type: DataType,+ tokenizer: Arc<Tokenizer>,+ token_codec_config: AnyTokenCodecConfig,}#[derive(Debug, Error)]@@ -39,6 +42,8 @@ pub enum EngineLoadClassifierError<B: Backend> {ParameterLoader(#[from] ParameterLoaderError<B>),#[error("Classifier error: {0}")]Classifier(#[from] ClassifierEncodableError<B>),+ #[error("Tokenizer error: {0}")]+ Tokenizer(#[from] tokenizers::Error),}impl<B: Backend> Engine<B> {@@ -72,11 +77,16 @@ impl<B: Backend> Engine<B> {weight_loader.tree().assert_all_tensors_validated()?;+ let tokenizer = Arc::new(Tokenizer::from_file(model_path.join("tokenizer.json"))?);+ let token_codec_config = config.token_codec_config;+Ok(ClassifierModel {context,classifier,output_labels,data_type,+ tokenizer,+ token_codec_config,})}}@@ -94,6 +104,14 @@ pub enum ClassifierModelClassifyError<B: Backend> {}impl<B: Backend> ClassifierModel<B> {+ pub fn tokenizer(&self) -> &Arc<Tokenizer> {+ &self.tokenizer+ }++ pub fn token_codec_config(&self) -> &AnyTokenCodecConfig {+ &self.token_codec_config+ }+pub fn classify(&self,input: &[u64],diff --git a/crates/backend-uzu/src/engine/language_model/mod.rs b/crates/backend-uzu/src/engine/language_model/mod.rsindex 6148e2379..94c5d315f 100644--- a/crates/backend-uzu/src/engine/language_model/mod.rs+++ b/crates/backend-uzu/src/engine/language_model/mod.rs@@ -1,6 +1,7 @@use std::{fs::File, io, io::BufReader, path::Path, sync::Arc};use thiserror::Error;+use tokenizers::Tokenizer;use crate::{backends::common::{Backend, Context, DeviceCapabilities, Kernels, kernel::ContextRingUpdateKernel},@@ -28,6 +29,7 @@ pub struct LanguageModel<B: Backend> {sampling: Sampling<B>,context_ring_update: <B::Kernels as Kernels>::ContextRingUpdateKernel,generation_config: GenerationConfig,+ tokenizer: Arc<Tokenizer>,#[cfg(grammar)]vocab_size: usize,}@@ -48,6 +50,8 @@ pub enum EngineLoadLanguageModelError<B: Backend> {Decoder(#[from] DecoderError<B>),#[error("Speculator error: {0}")]Speculator(#[from] DFlashSpeculatorLoadError<B>),+ #[error("Tokenizer error: {0}")]+ Tokenizer(#[from] tokenizers::Error),}impl<B: Backend> Engine<B> {@@ -65,30 +69,8 @@ impl<B: Backend> Engine<B> {let speculator_path = model_path.join("speculator");let speculator_path = speculator_path.exists().then_some(speculator_path);- self.build_language_model(config, &weight_loader, speculator_path.as_deref())- }-- // TODO: better design- pub fn load_language_model_random(- self: &Arc<Self>,- config_path: &Path,- header_path: &Path,- seed: u64,- ) -> Result<LanguageModel<B>, EngineLoadLanguageModelError<B>> {- let config: LanguageModelConfig = serde_json::from_reader(BufReader::new(File::open(config_path)?))?;-- let header_file = File::open(header_path)?;- let weight_loader = ParameterLoader::new_random(&header_file, &*self.context, seed)?;+ let tokenizer = Arc::new(Tokenizer::from_file(model_path.join("tokenizer.json"))?);- self.build_language_model(config, &weight_loader, None)- }-- fn build_language_model(- self: &Arc<Self>,- config: LanguageModelConfig,- weight_loader: &ParameterLoader<B>,- speculator_path: Option<&Path>,- ) -> Result<LanguageModel<B>, EngineLoadLanguageModelError<B>> {let data_type = DataType::BF16;let decoder = Decoder::new(@@ -104,6 +86,7 @@ impl<B: Backend> Engine<B> {);let speculator = speculator_path+ .as_deref().map(|speculator_path| DFlashTfmSpeculator::new(speculator_path, self.context.clone())).transpose()?;@@ -126,6 +109,7 @@ impl<B: Backend> Engine<B> {sampling,context_ring_update,generation_config,+ tokenizer,#[cfg(grammar)]vocab_size,})@@ -182,4 +166,8 @@ impl<B: Backend> LanguageModel<B> {pub fn generation_config(&self) -> &GenerationConfig {&self.generation_config}++ pub fn tokenizer(&self) -> &Arc<Tokenizer> {+ &self.tokenizer+ }}diff --git a/crates/backend-uzu/src/parameters/loader.rs b/crates/backend-uzu/src/parameters/loader.rsindex 31e7e564e..807c4ee1e 100644--- a/crates/backend-uzu/src/parameters/loader.rs+++ b/crates/backend-uzu/src/parameters/loader.rs@@ -4,8 +4,6 @@ use std::{fs::File,};-use half::{bf16, f16};-use rand::{RngExt, SeedableRng, rngs::SmallRng};use thiserror::Error;use super::safetensors_metadata::{HeaderLoadingError, read_metadata as read_st_metadata};@@ -54,17 +52,12 @@ pub enum ParameterLoaderError<B: Backend> {},}-enum ParameterBytes<'a> {- File(&'a File),- Random(u64),-}-pub struct ParameterLoader<'a, B: Backend> {context: &'a B::Context,index: HashMap<String, ParameterMetadata>,metadata: HashMap<String, String>,validated_tensors: RefCell<HashSet<String>>,- bytes: ParameterBytes<'a>,+ file: &'a File,}impl<'a, B: Backend> ParameterLoader<'a, B> {@@ -72,23 +65,7 @@ impl<'a, B: Backend> ParameterLoader<'a, B> {file: &'a File,context: &'a B::Context,) -> Result<Self, HeaderLoadingError> {- Self::from_header(file, context, ParameterBytes::File(file))- }-- pub fn new_random(- header_file: &File,- context: &'a B::Context,- seed: u64,- ) -> Result<Self, HeaderLoadingError> {- Self::from_header(header_file, context, ParameterBytes::Random(seed))- }-- fn from_header(- header_file: &File,- context: &'a B::Context,- bytes: ParameterBytes<'a>,- ) -> Result<Self, HeaderLoadingError> {- let (global_offset, st_metadata) = read_st_metadata(header_file)?;+ let (global_offset, st_metadata) = read_st_metadata(file)?;let index = st_metadata.tensors.into_iter()@@ -121,7 +98,7 @@ impl<'a, B: Backend> ParameterLoader<'a, B> {index,metadata,validated_tensors: RefCell::new(HashSet::new()),- bytes,+ file,})}@@ -178,12 +155,7 @@ impl<'a, 'leaf, B: Backend> ParameterLeaf<'a, 'leaf, B, true> {let element_count = self.metadata.size / std::mem::size_of::<T>();let mut data = vec![T::zeroed(); element_count];let destination = bytemuck::cast_slice_mut(&mut data);- match &self.loader.bytes {- ParameterBytes::File(file) => {- file_read_exact_at(file, destination, self.metadata.offset as u64)?;- },- ParameterBytes::Random(seed) => fill_random(destination, self.metadata.data_type, *seed),- }+ file_read_exact_at(self.loader.file, destination, self.metadata.offset as u64)?;Ok(data.into_boxed_slice())}@@ -201,60 +173,11 @@ impl<'a, 'leaf, B: Backend> ParameterLeaf<'a, 'leaf, B, true> {range.len(),)};- match &self.loader.bytes {- ParameterBytes::File(file) => {- file_read_exact_at(file, destination, self.metadata.offset as u64)?;- },- ParameterBytes::Random(seed) => fill_random(destination, self.metadata.data_type, *seed),- }+ file_read_exact_at(self.loader.file, destination, self.metadata.offset as u64)?;Ok(allocation)}}-fn fill_random(- destination: &mut [u8],- data_type: DataType,- seed: u64,-) {- if destination.is_empty() {- return;- }- let mut rng = SmallRng::seed_from_u64(seed);- let head_len = destination.len().min(65536);- let (head, tail) = destination.split_at_mut(head_len);- match data_type {- DataType::BF16 => {- for chunk in head.as_chunks_mut::<2>().0 {- *chunk = bf16::from_f32(rng.random_range(-0.1f32..0.1f32)).to_le_bytes();- }- },- DataType::F16 => {- for chunk in head.as_chunks_mut::<2>().0 {- *chunk = f16::from_f32(rng.random_range(-0.1f32..0.1f32)).to_le_bytes();- }- },- DataType::F32 => {- for chunk in head.as_chunks_mut::<4>().0 {- *chunk = rng.random_range(-0.1f32..0.1f32).to_le_bytes();- }- },- DataType::F64 => {- for chunk in head.as_chunks_mut::<8>().0 {- *chunk = rng.random_range(-0.1f64..0.1f64).to_le_bytes();- }- },- _ => {- for chunk in head.chunks_mut(8) {- let bytes = rng.random::<u64>().to_le_bytes();- chunk.copy_from_slice(&bytes[..chunk.len()]);- }- },- }- for target in tail.chunks_mut(head_len) {- target.copy_from_slice(&head[..target.len()]);- }-}-pub struct ParameterTree<'loader, B: Backend> {loader: &'loader ParameterLoader<'loader, B>,prefix: Option<String>,diff --git a/crates/hanashi/src/chat/hanashi/error.rs b/crates/hanashi/src/chat/hanashi/error.rsindex 839b5b7ba..9d6bbdc8b 100644--- a/crates/hanashi/src/chat/hanashi/error.rs+++ b/crates/hanashi/src/chat/hanashi/error.rs@@ -12,8 +12,6 @@ pub enum Error {ConfigNotFound(String),#[error("Config '{0}' is invalid")]InvalidConfig(String),- #[error("Unable to load tokenizer")]- UnableToLoadTokenizer,#[error("Unable to encode text")]UnableToEncodeText,#[error("Unable to decode token")]diff --git a/crates/hanashi/src/classification/config.rs b/crates/hanashi/src/classification/config.rsdeleted file mode 100644index 059ff78cb..000000000--- a/crates/hanashi/src/classification/config.rs+++ /dev/null@@ -1,39 +0,0 @@-use std::{io, io::ErrorKind};--use serde_json::Value;--pub enum TokenCodecConfig {- Chat(ChatTokenCodecConfig),- RawText,-}--impl TokenCodecConfig {- pub fn from_value(value: Value) -> Result<Self, io::Error> {- let typ = value.get("type");- match typ {- Some(Value::String(codec_type)) if codec_type == "RawTextCodecConfig" => Ok(Self::RawText),- Some(Value::String(codec_type)) if codec_type == "ChatCodecConfig" => {- let config = serde_json::from_value::<ChatTokenCodecConfig>(value)?;- Ok(Self::Chat(config))- },- None => {- let config = serde_json::from_value::<ChatTokenCodecConfig>(value)?;- Ok(Self::Chat(config))- },- _ => Err(io::Error::new(ErrorKind::InvalidData, format!("Invalid config: unknown type \"{typ:?}\""))),- }- }-}--#[derive(Debug, Clone, PartialEq, ::serde::Serialize, ::serde::Deserialize)]-pub struct ChatTokenCodecConfig {- pub prompt_template: String,- pub output_parser_regex: Option<String>,- pub system_role_name: String,- pub user_role_name: String,- pub assistant_role_name: String,- pub eos_token: Option<String>,- pub bos_token: Option<String>,- pub end_of_thinking_tag: Option<String>,- pub default_system_prompt: Option<String>,-}diff --git a/crates/hanashi/src/classification/encoding.rs b/crates/hanashi/src/classification/encoding.rsindex 8060080fc..5edfbda33 100644--- a/crates/hanashi/src/classification/encoding.rs+++ b/crates/hanashi/src/classification/encoding.rs@@ -1,56 +1,37 @@-use std::{collections::HashMap, fs::File, path::PathBuf, string::ToString};+use std::{collections::HashMap, string::ToString, sync::Arc};use minijinja::{Environment, context};use minijinja_contrib::pycompat::unknown_method_callback;-use serde_json::Value;-use shoji::types::session::classification::{ClassificationMessage, ClassificationRole};+use shoji::types::session::classification::{+ ChatTokenCodecConfig, ClassificationMessage, ClassificationRole, TokenCodecConfig,+};use tokenizers::Tokenizer;-use crate::{- chat::{- TokenizerLocation,- hanashi::{- Error,- renderer::{- RAISE_EXCEPTION_FUNCTION_NAME, STRFTIME_NOW_FUNCTION_NAME, TEMPLATE_NAME, TOJSON_FILTER_NAME,- raise_exception, to_json,- },+use crate::chat::{+ hanashi::{+ Error,+ renderer::{+ RAISE_EXCEPTION_FUNCTION_NAME, STRFTIME_NOW_FUNCTION_NAME, TEMPLATE_NAME, TOJSON_FILTER_NAME,+ raise_exception, to_json,},- strftime_now,},- classification::config::{ChatTokenCodecConfig, TokenCodecConfig},- util::tokenizer::load_tokenizer,+ strftime_now,};-const CONFIG_FILE_NAME: &str = "config.json";-pub struct ClassificationEncoding {- tokenizer: Tokenizer,+ tokenizer: Arc<Tokenizer>,config: TokenCodecConfig,}impl ClassificationEncoding {- pub fn new(model_dir: &str) -> Result<Self, Error> {- let model_path = PathBuf::from(model_dir);- let config_file = File::open(model_path.join(CONFIG_FILE_NAME))- .map_err(|_| Error::ConfigNotFound(CONFIG_FILE_NAME.to_string()))?;- let config = serde_json::from_reader::<File, Value>(config_file)- .map_err(|_| Error::InvalidConfig(CONFIG_FILE_NAME.to_string()))?;- let codec_config =- config.get("token_codec_config").ok_or_else(|| Error::InvalidConfig(CONFIG_FILE_NAME.to_string()))?;- let config = TokenCodecConfig::from_value(codec_config.clone())- .map_err(|_| Error::InvalidConfig(CONFIG_FILE_NAME.to_string()))?;-- let tokenizer_location = TokenizerLocation::Directory {- path: model_dir.to_string(),- name: None,- };- let tokenizer = load_tokenizer(&tokenizer_location).map_err(|_| Error::UnableToLoadTokenizer)?;-- Ok(Self {+ pub fn new(+ config: TokenCodecConfig,+ tokenizer: Arc<Tokenizer>,+ ) -> Self {+ Self {tokenizer,config,- })+ }}pub fn encode(diff --git a/crates/hanashi/src/classification/mod.rs b/crates/hanashi/src/classification/mod.rsindex 6f2940021..295694271 100644--- a/crates/hanashi/src/classification/mod.rs+++ b/crates/hanashi/src/classification/mod.rs@@ -1,2 +1 @@-mod config;pub mod encoding;diff --git a/crates/hanashi/src/lib.rs b/crates/hanashi/src/lib.rsindex ecca3db9c..d767d6176 100644--- a/crates/hanashi/src/lib.rs+++ b/crates/hanashi/src/lib.rs@@ -1,10 +1,8 @@mod classification;-mod util;pub mod chat;pub use classification::encoding::ClassificationEncoding;-pub use util::tokenizer::load_tokenizer;pub trait Encoding {type Config;diff --git a/crates/hanashi/src/util/mod.rs b/crates/hanashi/src/util/mod.rsdeleted file mode 100644index d9b8fd121..000000000--- a/crates/hanashi/src/util/mod.rs+++ /dev/null@@ -1 +0,0 @@-pub mod tokenizer;diff --git a/crates/hanashi/src/util/tokenizer.rs b/crates/hanashi/src/util/tokenizer.rsdeleted file mode 100644index 522fc6d00..000000000--- a/crates/hanashi/src/util/tokenizer.rs+++ /dev/null@@ -1,23 +0,0 @@-use std::path::PathBuf;--use tokenizers::Tokenizer;--use crate::chat::TokenizerLocation;--const DEFAULT_TOKENIZER_FILENAME: &str = "tokenizer.json";--pub fn load_tokenizer(location: &TokenizerLocation) -> Result<Tokenizer, Box<dyn std::error::Error + Send + Sync>> {- let tokenizer_path = match location {- TokenizerLocation::File {- path,- } => PathBuf::from(path),- TokenizerLocation::Directory {- path,- name,- } => match name {- Some(name) => PathBuf::from(path).join(name),- None => PathBuf::from(path).join(DEFAULT_TOKENIZER_FILENAME),- },- };- Tokenizer::from_file(&tokenizer_path)-}diff --git a/crates/nagare/src/chat/token.rs b/crates/nagare/src/chat/token.rsindex 33a59e7ee..569571712 100644--- a/crates/nagare/src/chat/token.rs+++ b/crates/nagare/src/chat/token.rs@@ -1,7 +1,6 @@use std::{io,pin::Pin,- sync::Arc,time::{Duration, Instant},};@@ -9,7 +8,6 @@ use futures::{Stream, StreamExt, stream};use hanashi::{Encoding as EncodingTrait,chat::{Encoding, EncodingConfig, TokenizerLocation, hanashi::HanashiEncodingImpl, harmony::HarmonyEncodingImpl},- load_tokenizer,};use shoji::{traits::{@@ -81,15 +79,16 @@ impl Session {path: reference.clone(),name: None,};- let tokenizer = load_tokenizer(&tokenizer_location).map_err(|err| ChatSessionError::Loading {- message: format!("Failed to initialize tokenizer: {err}"),- })?;- let tokenizer = Arc::new(tokenizer);++ let instance =+ backend.instance(reference.clone(), config).await.map_err(|error| ChatSessionError::Backend {+ message: error.to_string(),+ })?;let encoding = match encoding_config {Some(EncodingConfig::Hanashi {config,- }) => HanashiEncodingImpl::new(config, tokenizer.clone()).map(Encoding::Hanashi).map_err(|err| {+ }) => HanashiEncodingImpl::new(config, instance.tokenizer()).map(Encoding::Hanashi).map_err(|err| {ChatSessionError::Loading {message: format!("can not create harmony encoding: {err}"),}@@ -105,12 +104,6 @@ impl Session {message: "can not get encoding config".to_string(),}),}?;-- let instance = backend.instance(reference.clone(), config, tokenizer).await.map_err(|error| {- ChatSessionError::Backend {- message: error.to_string(),- }- })?;let state = instance.state().await.map_err(|error| ChatSessionError::Backend {message: error.to_string(),})?;diff --git a/crates/nagare/src/classification/mod.rs b/crates/nagare/src/classification/mod.rsindex 312a1ef2f..7c159a3fa 100644--- a/crates/nagare/src/classification/mod.rs+++ b/crates/nagare/src/classification/mod.rs@@ -63,10 +63,7 @@ impl ClassificationSession {let instance_state = instance.state().await.map_err(|error| ClassificationSessionError::Backend {message: error.to_string(),})?;- let encoding =- ClassificationEncoding::new(reference.as_str()).map_err(|err| ClassificationSessionError::Backend {- message: err.to_string(),- })?;+ let encoding = ClassificationEncoding::new(instance.token_codec_config(), instance.tokenizer());Ok(InstanceHolder {instance,diff --git a/crates/shoji/src/traits/backend/chat_token.rs b/crates/shoji/src/traits/backend/chat_token.rsindex 5aa3ce9cf..f77d28f89 100644--- a/crates/shoji/src/traits/backend/chat_token.rs+++ b/crates/shoji/src/traits/backend/chat_token.rs@@ -31,7 +31,6 @@ pub trait Backend: Send + Sync {&'a self,reference: String,config: ChatConfig,- tokenizer: Arc<Tokenizer>,) -> Pin<Box<dyn Future<Output = Result<Box<dyn Instance>, Error>> + Send + 'a>>;}@@ -43,6 +42,8 @@ pub trait Instance:StreamMetrics = StreamMetrics,>{+ fn tokenizer(&self) -> Arc<Tokenizer>;+fn max_context_length(&self) -> Option<usize>;fn stop_token_ids(&self) -> Option<Box<[u64]>>;diff --git a/crates/shoji/src/traits/backend/classification.rs b/crates/shoji/src/traits/backend/classification.rsindex 8111639db..3f529bba6 100644--- a/crates/shoji/src/traits/backend/classification.rs+++ b/crates/shoji/src/traits/backend/classification.rs@@ -1,6 +1,11 @@-use std::{collections::HashMap, convert::Infallible, pin::Pin};+use std::{collections::HashMap, convert::Infallible, pin::Pin, sync::Arc};-use crate::traits::backend::{Error, Instance as InstanceTrait};+use tokenizers::Tokenizer;++use crate::{+ traits::backend::{Error, Instance as InstanceTrait},+ types::session::classification::TokenCodecConfig,+};pub struct ClassifierOutput {pub logits: Vec<f32>,@@ -29,14 +34,7 @@ pub trait Instance:StreamMetrics = StreamMetrics,>{-}+ fn tokenizer(&self) -> Arc<Tokenizer>;-impl<T> Instance for T where- T: InstanceTrait<- StreamConfig = StreamConfig,- StreamInput = StreamInput,- StreamOutput = StreamOutput,- StreamMetrics = StreamMetrics,- >-{+ fn token_codec_config(&self) -> TokenCodecConfig;}diff --git a/crates/shoji/src/types/session/classification/codec.rs b/crates/shoji/src/types/session/classification/codec.rsnew file mode 100644index 000000000..15ba61993--- /dev/null+++ b/crates/shoji/src/types/session/classification/codec.rs@@ -0,0 +1,20 @@+use serde::{Deserialize, Serialize};++#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]+pub enum TokenCodecConfig {+ Chat(ChatTokenCodecConfig),+ RawText,+}++#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]+pub struct ChatTokenCodecConfig {+ pub prompt_template: String,+ pub output_parser_regex: Option<String>,+ pub system_role_name: String,+ pub user_role_name: String,+ pub assistant_role_name: String,+ pub eos_token: Option<String>,+ pub bos_token: Option<String>,+ pub end_of_thinking_tag: Option<String>,+ pub default_system_prompt: Option<String>,+}diff --git a/crates/shoji/src/types/session/classification/mod.rs b/crates/shoji/src/types/session/classification/mod.rsindex 1b63fb030..6c9c559e3 100644--- a/crates/shoji/src/types/session/classification/mod.rs+++ b/crates/shoji/src/types/session/classification/mod.rs@@ -1,9 +1,11 @@+mod codec;mod label;mod message;mod output;mod role;mod stats;+pub use codec::{ChatTokenCodecConfig, TokenCodecConfig};pub use label::ClassificationLabel;pub use message::ClassificationMessage;pub use output::{ClassificationOutput, ClassificationOutputProbabilities};