💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Fix cached_path error, closes #340
davidmezzetti33bc4261be20
diff --git a/setup.py b/setup.pyindex eb3d5f77..c5736511 100644--- a/setup.py+++ b/setup.py@@ -6,7 +6,7 @@ with open("README.md", "r", encoding="utf-8") as f:DESCRIPTION = "".join([line for line in f if "gh-dark-mode-only" not in line])# Required dependencies-install = ["faiss-cpu>=1.7.1.post2", "numpy>=1.18.4", "pyyaml>=5.3", "torch>=1.6.0", "transformers>=4.20.1"]+install = ["faiss-cpu>=1.7.1.post2", "numpy>=1.18.4", "pyyaml>=5.3", "torch>=1.6.0", "transformers>=4.22.0"]# Optional dependenciesextras = {}diff --git a/src/python/txtai/pipeline/text/translation.py b/src/python/txtai/pipeline/text/translation.pyindex b4dc6422..4e7a76c8 100644--- a/src/python/txtai/pipeline/text/translation.py+++ b/src/python/txtai/pipeline/text/translation.py@@ -10,9 +10,9 @@ try:except ImportError:FASTTEXT = False+from huggingface_hub import cached_downloadfrom huggingface_hub.hf_api import HfApifrom transformers import M2M100ForConditionalGeneration, M2M100Tokenizer, MarianMTModel, MarianTokenizer-from transformers.file_utils import cached_pathfrom ..hfmodel import HFModel@@ -121,7 +121,7 @@ class Translation(HFModel):fasttext.FastText.eprint = lambda x: None# Load language detection model- path = cached_path(self.langdetect)+ path = cached_download(self.langdetect, legacy_cache_layout=True)self.detector = fasttext.load_model(path)# Transform texts to format expected by language detection model@@ -160,10 +160,10 @@ class Translation(HFModel):tokenizer.src_lang = sourcetokens, indices = self.tokenize(tokenizer, texts)- translated = model.generate(**tokens, forced_bos_token_id=tokenizer.lang_code_to_id[target])+ translated = model.generate(**tokens, forced_bos_token_id=tokenizer.lang_code_to_id[target], max_length=model.config.max_length)else:tokens, indices = self.tokenize(tokenizer, texts)- translated = model.generate(**tokens)+ translated = model.generate(**tokens, max_length=model.config.max_length)# Decode translationstranslated = tokenizer.batch_decode(translated, skip_special_tokens=True)