Skip to content

[KDD'2026] "VideoRAG: Chat with Your Videos"

feat:add deepseek and bge support

zqlc6ba87dd2593
diff --git a/process_videos.py b/process_videos.py
new file mode 100644
index 0000000..535502f
--- /dev/null
+++ b/process_videos.py
@@ -0,0 +1,30 @@
+import os
+import logging
+import warnings
+import multiprocessing
+
+warnings.filterwarnings("ignore")
+logging.getLogger("httpx").setLevel(logging.WARNING)
+
+# 设置DeepSeek和硅基流动的API密钥
+os.environ["DEEPSEEK_API_KEY"] = "sk-*******"
+os.environ["SILICONFLOW_API_KEY"] = "sk-******"
+
+from videorag._llm import deepseek_bge_config
+from videorag import VideoRAG, QueryParam
+
+
+if __name__ == '__main__':
+ # 必须的设置
+ multiprocessing.set_start_method('spawn')
+
+ # 将你的视频文件路径放入这个列表中
+ video_paths = [
+ '/home/zy/VideoRAG/fc6207139422fa5f030113b7d79efe5e.mp4',
+ ]
+
+ # 初始化 VideoRAG,指定一个工作目录来存放索引文件
+ videorag = VideoRAG(llm=deepseek_bge_config, working_dir=f"./videorag-workdir")
+
+ # 开始处理视频
+ videorag.insert_video(video_path_list=video_paths)
\ No newline at end of file
diff --git a/query_videos.py b/query_videos.py
new file mode 100644
index 0000000..4fba653
--- /dev/null
+++ b/query_videos.py
@@ -0,0 +1,36 @@
+import os
+import logging
+import warnings
+import multiprocessing
+
+warnings.filterwarnings("ignore")
+logging.getLogger("httpx").setLevel(logging.WARNING)
+
+# 设置DeepSeek和硅基流动的API密钥
+os.environ["DEEPSEEK_API_KEY"] = "sk-*******"
+os.environ["SILICONFLOW_API_KEY"] = "sk-*******"
+
+from videorag._llm import deepseek_bge_config
+from videorag import VideoRAG, QueryParam
+
+
+if __name__ == '__main__':
+ # 必须的设置
+ multiprocessing.set_start_method('spawn')
+
+ # 你想问的问题
+ query = '请描述两个视频的主要内容是什么?'
+
+ param = QueryParam(mode="videorag")
+ # 如果设置为 False,返回的答案会附带视频片段的引用
+ param.wo_reference = True
+
+ # 初始化 VideoRAG,并确保工作目录与上一步骤中的一致
+ videorag = VideoRAG(llm=deepseek_bge_config, working_dir=f"./videorag-workdir")
+ videorag.load_caption_model(debug=False)
+
+ # 执行查询
+ response = videorag.query(query=query, param=param)
+
+ # 打印结果
+ print(response)
\ No newline at end of file
diff --git a/videorag/_llm.py b/videorag/_llm.py
index 2b23d1f..ca53f5e 100755
--- a/videorag/_llm.py
+++ b/videorag/_llm.py
@@ -387,3 +387,114 @@ ollama_config = LLMConfig(
cheap_model_max_token_size = 32768,
cheap_model_max_async = 1
)
+###### DeepSeek Configuration
+@retry(
+ stop=stop_after_attempt(5),
+ wait=wait_exponential(multiplier=1, min=4, max=10),
+ retry=retry_if_exception_type((RateLimitError, APIConnectionError)),
+)
+async def deepseek_complete_if_cache(
+ model, prompt, system_prompt=None, history_messages=[], **kwargs
+) -> str:
+ # 使用DeepSeek API
+ import httpx
+
+ hashing_kv: BaseKVStorage = kwargs.pop("hashing_kv", None)
+ messages = []
+ if system_prompt:
+ messages.append({"role": "system", "content": system_prompt})
+ messages.extend(history_messages)
+ messages.append({"role": "user", "content": prompt})
+
+ if hashing_kv is not None:
+ args_hash = compute_args_hash(model, messages)
+ if_cache_return = await hashing_kv.get_by_id(args_hash)
+ if if_cache_return is not None and if_cache_return["return"] is not None:
+ return if_cache_return["return"]
+
+ # DeepSeek API调用
+ async with httpx.AsyncClient() as client:
+ response = await client.post(
+ "https://api.deepseek.com/v1/chat/completions",
+ headers={
+ "Authorization": f"Bearer {os.environ.get('DEEPSEEK_API_KEY', 'sk-*******')}",
+ "Content-Type": "application/json"
+ },
+ json={
+ "model": model,
+ "messages": messages,
+ "temperature": kwargs.get("temperature", 0.7),
+ "max_tokens": kwargs.get("max_tokens", 4096)
+ },
+ timeout=60.0
+ )
+ response.raise_for_status()
+ result = response.json()
+ content = result["choices"][0]["message"]["content"]
+
+ if hashing_kv is not None:
+ await hashing_kv.upsert(
+ {args_hash: {"return": content, "model": model}}
+ )
+ await hashing_kv.index_done_callback()
+
+ return content
+
+async def deepseek_complete(model_name, prompt, system_prompt=None, history_messages=[], **kwargs) -> str:
+ return await deepseek_complete_if_cache(
+ model_name,
+ prompt,
+ system_prompt=system_prompt,
+ history_messages=history_messages,
+ **kwargs
+ )
+
+@retry(
+ stop=stop_after_attempt(5),
+ wait=wait_exponential(multiplier=1, min=4, max=10),
+ retry=retry_if_exception_type((RateLimitError, APIConnectionError)),
+)
+async def bge_m3_embedding(model_name: str, texts: list[str]) -> np.ndarray:
+ # 使用硅基流动的BAAI/bge-m3嵌入模型
+ import httpx
+
+ async with httpx.AsyncClient() as client:
+ response = await client.post(
+ "https://api.siliconflow.cn/v1/embeddings",
+ headers={
+ "Authorization": f"Bearer {os.environ.get('SILICONFLOW_API_KEY', 'sk-******')}",
+ "Content-Type": "application/json"
+ },
+ json={
+ "model": "BAAI/bge-m3",
+ "input": texts,
+ "encoding_format": "float"
+ },
+ timeout=60.0
+ )
+ response.raise_for_status()
+ result = response.json()
+ embeddings = [item["embedding"] for item in result["data"]]
+ return np.array(embeddings)
+
+# DeepSeek + BAAI/bge-m3 配置
+deepseek_bge_config = LLMConfig(
+ embedding_func_raw = bge_m3_embedding,
+ embedding_model_name = "BAAI/bge-m3",
+ embedding_dim = 1024, # bge-m3的嵌入维度
+ embedding_max_token_size = 8192,
+ embedding_batch_num = 32,
+ embedding_func_max_async = 16,
+ query_better_than_threshold = 0.2,
+
+ best_model_func_raw = deepseek_complete,
+ best_model_name = "deepseek-chat",
+ best_model_max_token_size = 32768,
+ best_model_max_async = 16,
+
+ cheap_model_func_raw = deepseek_complete,
+ cheap_model_name = "deepseek-chat",
+ cheap_model_max_token_size = 32768,
+ cheap_model_max_async = 16
+)
+