Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(rag): log swallowed embedding/image progress and LightRAG meta failures (#1703)

evan188199-techcc373762e42f
diff --git a/deeptutor/services/embedding/client.py b/deeptutor/services/embedding/client.py
index 641cc4a13..adf6dbc48 100644
--- a/deeptutor/services/embedding/client.py
+++ b/deeptutor/services/embedding/client.py
@@ -201,8 +201,15 @@ class EmbeddingClient:
if progress_callback:
try:
progress_callback(i + 1, total_batches)
- except Exception:
- pass
+ except Exception as exc:
+ # Progress feedback is best-effort: never fail the embedding
+ # run, but never lose the failure silently either — index
+ # progress would read as stalled with no trace why.
+ self.logger.warning(
+ f"Embedding progress callback failed "
+ f"(batch {i + 1}/{total_batches}): {exc}",
+ exc_info=True,
+ )
# Delay between batches to avoid rate limiting
if i < total_batches - 1 and batch_delay > 0:
@@ -276,8 +283,14 @@ class EmbeddingClient:
if progress_callback:
try:
progress_callback(i + 1, total_batches)
- except Exception:
- pass
+ except Exception as exc:
+ # Same contract as embed(): log and continue, so a broken
+ # progress sink cannot stall multimodal indexing invisibly.
+ self.logger.warning(
+ f"Embedding progress callback failed "
+ f"(batch {i + 1}/{total_batches}): {exc}",
+ exc_info=True,
+ )
if i < total_batches - 1 and self.config.batch_delay > 0:
await asyncio.sleep(self.config.batch_delay)
diff --git a/deeptutor/services/rag/pipelines/lightrag/engine.py b/deeptutor/services/rag/pipelines/lightrag/engine.py
index 1022246ef..595797064 100644
--- a/deeptutor/services/rag/pipelines/lightrag/engine.py
+++ b/deeptutor/services/rag/pipelines/lightrag/engine.py
@@ -8,6 +8,7 @@ import hashlib
from importlib.metadata import PackageNotFoundError, version
import inspect
import json
+import logging
from pathlib import Path
import re
from typing import TYPE_CHECKING, Any
@@ -32,6 +33,8 @@ LIGHTRAG_DISTRIBUTION = "lightrag-hku"
LIGHTRAG_VERSION = "1.5.7"
PARSER_ENGINE = "deeptutor"
+logger = logging.getLogger(__name__)
+
class LightRagContractError(RuntimeError):
"""Raised when the pinned SDK returns a shape DeepTutor cannot trust."""
@@ -187,8 +190,9 @@ def workspace_for(working_dir: Path) -> str:
# A published version keeps its native LightRAG workspace name when its
# containing KB is moved. The name is a hash of the original absolute
# version path; recomputing it after a move would open an empty store.
+ meta_path = root / "meta.json"
try:
- meta = json.loads((root / "meta.json").read_text(encoding="utf-8"))
+ meta = json.loads(meta_path.read_text(encoding="utf-8"))
published = str(meta.get("workspace") or "") if isinstance(meta, dict) else ""
if (
isinstance(meta, dict)
@@ -196,8 +200,16 @@ def workspace_for(working_dir: Path) -> str:
and re.fullmatch(r"deeptutor_[0-9a-f]{16}", published)
):
return published
- except (OSError, ValueError):
+ except FileNotFoundError:
pass
+ except (OSError, ValueError) as exc:
+ logger.warning(
+ "LightRAG workspace meta at %s exists but cannot be used (%s: %s); "
+ "falling back to path-hash workspace name, which may open an empty store",
+ meta_path,
+ type(exc).__name__,
+ exc,
+ )
identity = str(root.resolve()).encode("utf-8")
return f"deeptutor_{hashlib.sha256(identity).hexdigest()[:16]}"
diff --git a/deeptutor/services/rag/pipelines/llamaindex/document_loader.py b/deeptutor/services/rag/pipelines/llamaindex/document_loader.py
index 7f662cfd5..a237ea139 100644
--- a/deeptutor/services/rag/pipelines/llamaindex/document_loader.py
+++ b/deeptutor/services/rag/pipelines/llamaindex/document_loader.py
@@ -402,8 +402,13 @@ class LlamaIndexDocumentLoader:
if image_progress_callback:
try:
image_progress_callback(completed, total)
- except Exception:
- pass
+ except Exception as exc:
+ self.logger.warning(
+ "Image progress callback failed (completed=%s, total=%s): %s",
+ completed,
+ total,
+ exc,
+ )
return result
# gather preserves input order, so embedded/descriptions/contents stay
diff --git a/tests/services/embedding/test_client_runtime.py b/tests/services/embedding/test_client_runtime.py
index fc45e5924..44f8606cf 100644
--- a/tests/services/embedding/test_client_runtime.py
+++ b/tests/services/embedding/test_client_runtime.py
@@ -3,6 +3,7 @@
from __future__ import annotations
import asyncio
+import logging
import threading
from typing import Any
@@ -368,3 +369,65 @@ def test_embedding_client_multimodal_detection_uses_model_level_metadata() -> No
assert text_model.supports_multimodal_contents() is False
assert vision_model.supports_multimodal_contents() is True
assert cohere_v3.supports_multimodal_contents() is False
+
+
+async def test_embed_progress_callback_failure_logged_and_not_fatal(monkeypatch, caplog) -> None:
+ """A failing progress callback must not break embedding, but the failure
+ cannot stay silent — swallowed errors make KB index progress untrustworthy."""
+
+ def _broken_callback(completed: int, total: int) -> None:
+ raise RuntimeError("progress sink offline")
+
+ _FakeAdapter.instances = []
+ monkeypatch.setattr(
+ "deeptutor.services.embedding.client._resolve_adapter_class", lambda _b: _FakeAdapter
+ )
+ client = EmbeddingClient(_build_config("openai"))
+
+ with caplog.at_level(logging.WARNING, logger="deeptutor.services.embedding.client"):
+ vectors = await client.embed(["a", "b", "c"], progress_callback=_broken_callback)
+
+ assert len(vectors) == 3
+ warnings = [r for r in caplog.records if r.levelno >= logging.WARNING]
+ assert any("progress callback" in r.getMessage() for r in warnings)
+ assert any("progress sink offline" in r.getMessage() for r in warnings)
+
+
+async def test_embed_contents_progress_callback_failure_logged_and_not_fatal(
+ monkeypatch, caplog
+) -> None:
+ class _MultimodalAdapter(_FakeAdapter):
+ def get_model_info(self):
+ return {"multimodal": True}
+
+ async def embed(self, request):
+ self.calls.append(request)
+ items = request.contents or [{"text": t} for t in request.texts]
+ return type(
+ "Resp",
+ (),
+ {"embeddings": [[float(i)] * 2 for i, _ in enumerate(items)]},
+ )()
+
+ def _broken_callback(completed: int, total: int) -> None:
+ raise RuntimeError("progress sink offline")
+
+ _FakeAdapter.instances = []
+ monkeypatch.setattr(
+ "deeptutor.services.embedding.client._resolve_adapter_class",
+ lambda _b: _MultimodalAdapter,
+ )
+ client = EmbeddingClient(_build_config("openai"))
+
+ with caplog.at_level(logging.WARNING, logger="deeptutor.services.embedding.client"):
+ vectors = await client.embed_contents(
+ [{"text": "a"}, {"text": "b"}, {"text": "c"}],
+ progress_callback=_broken_callback,
+ )
+
+ assert len(vectors) == 3
+ warnings = [r for r in caplog.records if r.levelno >= logging.WARNING]
+ assert any("progress callback" in r.getMessage() for r in warnings)
+ assert any("progress sink offline" in r.getMessage() for r in warnings)
diff --git a/tests/services/rag/test_lightrag_workspace_meta.py b/tests/services/rag/test_lightrag_workspace_meta.py
new file mode 100644
index 000000000..6d461453c
--- /dev/null
+++ b/tests/services/rag/test_lightrag_workspace_meta.py
@@ -0,0 +1,72 @@
+"""Regression tests for workspace_for() meta.json failure handling."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+from pathlib import Path
+
+import pytest
+
+from deeptutor.services.rag.pipelines.lightrag import engine
+
+_ENGINE_LOGGER = "deeptutor.services.rag.pipelines.lightrag.engine"
+
+
+def _hash_workspace(path: Path) -> str:
+ identity = str(path.resolve()).encode("utf-8")
+ return f"deeptutor_{hashlib.sha256(identity).hexdigest()[:16]}"
+
+
+def test_corrupt_meta_json_warns_and_falls_back(
+ tmp_path: Path, caplog: pytest.LogCaptureFixture
+) -> None:
+ (tmp_path / "meta.json").write_text("{not valid json", encoding="utf-8")
+ with caplog.at_level(logging.WARNING, logger=_ENGINE_LOGGER):
+ name = engine.workspace_for(tmp_path)
+ assert name == _hash_workspace(tmp_path)
+ warnings = [r for r in caplog.records if r.levelno == logging.WARNING]
+ assert warnings, "expected a warning when meta.json exists but fails to parse"
+ message = warnings[0].getMessage()
+ assert "meta.json" in message
+ assert "JSONDecodeError" in message
+
+
+def test_unreadable_meta_json_warns_and_falls_back(
+ tmp_path: Path, caplog: pytest.LogCaptureFixture
+) -> None:
+ meta = tmp_path / "meta.json"
+ meta.write_text(
+ json.dumps({"provider": "lightrag", "workspace": "deeptutor_abc"}), encoding="utf-8"
+ )
+ meta.chmod(0o000)
+ with caplog.at_level(logging.WARNING, logger=_ENGINE_LOGGER):
+ name = engine.workspace_for(tmp_path)
+ assert name == _hash_workspace(tmp_path)
+ warnings = [r for r in caplog.records if r.levelno == logging.WARNING]
+ assert warnings, "expected a warning when meta.json exists but cannot be read"
+ message = warnings[0].getMessage()
+ assert "meta.json" in message
+ assert "PermissionError" in message
+
+
+def test_missing_meta_json_stays_silent(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None:
+ with caplog.at_level(logging.WARNING, logger=_ENGINE_LOGGER):
+ name = engine.workspace_for(tmp_path)
+ assert name == _hash_workspace(tmp_path)
+ assert [r for r in caplog.records if r.levelno >= logging.WARNING] == []
+
+
+def test_valid_meta_json_returns_published_name(
+ tmp_path: Path, caplog: pytest.LogCaptureFixture
+) -> None:
+ published = "deeptutor_" + "ab" * 8
+ (tmp_path / "meta.json").write_text(
+ json.dumps({"provider": "lightrag", "workspace": published}),
+ encoding="utf-8",
+ )
+ with caplog.at_level(logging.WARNING, logger=_ENGINE_LOGGER):
+ name = engine.workspace_for(tmp_path)
+ assert name == published
+ assert [r for r in caplog.records if r.levelno >= logging.WARNING] == []
diff --git a/tests/services/rag/test_llamaindex_document_loader.py b/tests/services/rag/test_llamaindex_document_loader.py
index e0fabedcc..799872f5c 100644
--- a/tests/services/rag/test_llamaindex_document_loader.py
+++ b/tests/services/rag/test_llamaindex_document_loader.py
@@ -531,6 +531,67 @@ def test_loader_embeds_images_with_qwen38_max_vision_capability(
assert captured["llm_kwargs"]["image_mime_type"] == "image/png"
+def test_loader_logs_and_continues_when_image_progress_callback_fails(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ pytest.importorskip("llama_index.core")
+ from llama_index.core.schema import ImageNode
+
+ from deeptutor.services.rag.pipelines.llamaindex import document_loader as loader_module
+
+ image_paths = []
+ for name in ("photo-1.png", "photo-2.png"):
+ path = tmp_path / name
+ path.write_bytes(b"\x89PNG\r\n")
+ image_paths.append(path)
+
+ class _MultimodalEmbeddingClient:
+ config = type("Config", (), {"binding": "siliconflow", "model": "qwen3-vl"})()
+
+ def supports_multimodal_contents(self) -> bool:
+ return True
+
+ async def embed_contents(self, contents):
+ return [[0.1, 0.2, 0.3] for _ in contents]
+
+ class _VisionClient:
+ config = type("Config", (), {"binding": "openai", "model": "gpt-4o"})()
+
+ def supports_multimodal_images(self) -> bool:
+ return True
+
+ async def complete(self, prompt, **kwargs):
+ return "Stub description."
+
+ monkeypatch.setattr(loader_module, "get_embedding_client", lambda: _MultimodalEmbeddingClient())
+ monkeypatch.setattr(loader_module, "get_llm_client", lambda: _VisionClient())
+
+ calls: list[tuple[int, int]] = []
+
+ def _flaky_callback(completed: int, total: int) -> None:
+ calls.append((completed, total))
+ if completed == 1:
+ raise RuntimeError("progress sink is down")
+
+ with caplog.at_level("WARNING"):
+ documents = asyncio.run(
+ loader_module.LlamaIndexDocumentLoader().load(
+ [str(path) for path in image_paths],
+ image_progress_callback=_flaky_callback,
+ )
+ )
+
+ # The callback stays invoked for every image even after one failure.
+ assert calls == [(1, 2), (2, 2)]
+ # A failing callback must not abort the batch: both images still load.
+ assert len(documents) == 2
+ assert all(isinstance(doc, ImageNode) for doc in documents)
+ assert "Image progress callback failed" in caplog.text
+ assert "progress sink is down" in caplog.text
+
+
def test_loader_skips_images_when_llm_is_text_only(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None: