💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Update llama.cpp vectors to support default length and batch parameters, closes #983
davidmezzettibb37634f8c0a
diff --git a/src/python/txtai/vectors/dense/llama.py b/src/python/txtai/vectors/dense/llama.pyindex 9e5c7470..0cdcfb48 100644--- a/src/python/txtai/vectors/dense/llama.py+++ b/src/python/txtai/vectors/dense/llama.py@@ -10,7 +10,7 @@ from huggingface_hub import hf_hub_download# Conditional importtry:- from llama_cpp import Llama+ import llama_cpp as llamaLLAMA_CPP = Trueexcept ImportError:@@ -52,11 +52,20 @@ class LlamaCpp(Vectors):# Additional model argumentsmodelargs = self.config.get("vectors", {})+ # Default n_ctx to maxlength if available. Otherwise default n_ctx=0, which sets n_ctx=n_ctx_train.+ modelargs["n_ctx"] = modelargs.get("n_ctx", self.config.get("maxlength", 0))++ # Default n_batch to encode batch+ modelargs["n_batch"] = modelargs.get("n_batch", self.config.get("encodebatch", 64))+# Default GPU layers if not already setmodelargs["n_gpu_layers"] = modelargs.get("n_gpu_layers", -1 if self.config.get("gpu", os.environ.get("LLAMA_NO_METAL") != "1") else 0)+ # Default verbose flag+ modelargs["verbose"] = modelargs.get("verbose", False)+# Create llama.cpp instance- return Llama(path, n_ctx=0, verbose=modelargs.pop("verbose", False), embedding=True, **modelargs)+ return llama.Llama(model_path=path, embedding=True, **modelargs)def encode(self, data, category=None):# Generate embeddings and return as a NumPy array