Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Update llama.cpp vectors to support default length and batch parameters, closes #983

davidmezzettibb37634f8c0a
diff --git a/src/python/txtai/vectors/dense/llama.py b/src/python/txtai/vectors/dense/llama.py
index 9e5c7470..0cdcfb48 100644
--- a/src/python/txtai/vectors/dense/llama.py
+++ b/src/python/txtai/vectors/dense/llama.py
@@ -10,7 +10,7 @@ from huggingface_hub import hf_hub_download
# Conditional import
try:
- from llama_cpp import Llama
+ import llama_cpp as llama
LLAMA_CPP = True
except ImportError:
@@ -52,11 +52,20 @@ class LlamaCpp(Vectors):
# Additional model arguments
modelargs = self.config.get("vectors", {})
+ # Default n_ctx to maxlength if available. Otherwise default n_ctx=0, which sets n_ctx=n_ctx_train.
+ modelargs["n_ctx"] = modelargs.get("n_ctx", self.config.get("maxlength", 0))
+
+ # Default n_batch to encode batch
+ modelargs["n_batch"] = modelargs.get("n_batch", self.config.get("encodebatch", 64))
+
# Default GPU layers if not already set
modelargs["n_gpu_layers"] = modelargs.get("n_gpu_layers", -1 if self.config.get("gpu", os.environ.get("LLAMA_NO_METAL") != "1") else 0)
+ # Default verbose flag
+ modelargs["verbose"] = modelargs.get("verbose", False)
+
# Create llama.cpp instance
- return Llama(path, n_ctx=0, verbose=modelargs.pop("verbose", False), embedding=True, **modelargs)
+ return llama.Llama(model_path=path, embedding=True, **modelargs)
def encode(self, data, category=None):
# Generate embeddings and return as a NumPy array