Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Modify LiteLLM path detection logic, closes #947

davidmezzetti2956bd8789c0
diff --git a/src/python/txtai/pipeline/llm/litellm.py b/src/python/txtai/pipeline/llm/litellm.py
index 25db01b7..ee7faa8d 100644
--- a/src/python/txtai/pipeline/llm/litellm.py
+++ b/src/python/txtai/pipeline/llm/litellm.py
@@ -2,6 +2,8 @@
LiteLLM module
"""
+from transformers.utils import cached_file
+
# Conditional import
try:
import litellm as api
@@ -36,7 +38,7 @@ class LiteLLM(Generation):
try:
# Suppress debug messages for this test
api.suppress_debug_info = True
- return api.get_llm_provider(path)
+ return api.get_llm_provider(path) and not LiteLLM.ishub(path)
except:
return False
finally:
@@ -45,6 +47,24 @@ class LiteLLM(Generation):
return False
+ @staticmethod
+ def ishub(path):
+ """
+ Checks if path is available on the HF Hub.
+
+ Args:
+ input path
+
+ Returns:
+ True if this is a model on the HF Hub
+ """
+
+ # pylint: disable=W0702
+ try:
+ return cached_file(path_or_repo_id=path, filename="config.json") is not None if "/" in path else False
+ except:
+ return False
+
def __init__(self, path, template=None, **kwargs):
super().__init__(path, template, **kwargs)
diff --git a/src/python/txtai/vectors/dense/litellm.py b/src/python/txtai/vectors/dense/litellm.py
index ee6292ea..50a5b363 100644
--- a/src/python/txtai/vectors/dense/litellm.py
+++ b/src/python/txtai/vectors/dense/litellm.py
@@ -4,6 +4,8 @@ LiteLLM module
import numpy as np
+from transformers.utils import cached_file
+
# Conditional import
try:
import litellm as api
@@ -38,7 +40,7 @@ class LiteLLM(Vectors):
try:
# Suppress debug messages for this test
api.suppress_debug_info = True
- return api.get_llm_provider(path)
+ return api.get_llm_provider(path) and not LiteLLM.ishub(path)
except:
return False
finally:
@@ -47,6 +49,24 @@ class LiteLLM(Vectors):
return False
+ @staticmethod
+ def ishub(path):
+ """
+ Checks if path is available on the HF Hub.
+
+ Args:
+ input path
+
+ Returns:
+ True if this is a model on the HF Hub
+ """
+
+ # pylint: disable=W0702
+ try:
+ return cached_file(path_or_repo_id=path, filename="config.json") is not None if "/" in path else False
+ except:
+ return False
+
def __init__(self, config, scoring, models):
# Check before parent constructor since it calls loadmodel
if not LITELLM:
diff --git a/test/python/testpipeline/testllm/testlitellm.py b/test/python/testpipeline/testllm/testlitellm.py
index 688f6ab5..fca0c24a 100644
--- a/test/python/testpipeline/testllm/testlitellm.py
+++ b/test/python/testpipeline/testllm/testlitellm.py
@@ -102,7 +102,7 @@ class TestLiteLLM(unittest.TestCase):
"""
# Test model generation with LiteLLM
- model = LLM("gpt-4o", api_base="http://127.0.0.1:8000")
+ model = LLM("openai/gpt-4o", api_base="http://127.0.0.1:8000")
self.assertEqual(model("The sky is"), "blue")
# Test default role