Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(embeddings): say which pages lose text at the embedding cap (#1192)
Swati Ahuja2074fb6875cd
diff --git a/packages/core/src/repowise/core/persistence/vector_store/_base.py b/packages/core/src/repowise/core/persistence/vector_store/_base.pyindex e74490f9d..a038bd1fd 100644--- a/packages/core/src/repowise/core/persistence/vector_store/_base.py+++ b/packages/core/src/repowise/core/persistence/vector_store/_base.py@@ -8,6 +8,7 @@ re-exported from the package ``__init__`` so the historical import pathfrom __future__ import annotations+import loggingimport mathfrom abc import ABC, abstractmethodfrom collections.abc import Iterator@@ -22,6 +23,8 @@ __all__ = ["iter_embed_chunks",]+logger = logging.getLogger(__name__)+# One embedder call per chunk of this many items. OpenAI rejects embedding# requests past 300k total tokens — a generation level of 275 full wiki# pages (~560k tokens) failed in one giant request and silently lost the@@ -39,9 +42,34 @@ EMBED_TEXT_MAX_CHARS = 30_000def iter_embed_chunks(items: list[tuple[str, str, dict]],) -> Iterator[tuple[list[tuple[str, str, dict]], list[str]]]:- """Yield ``(chunk, capped_texts)`` slices sized for one embedder request."""+ """Yield ``(chunk, capped_texts)`` slices sized for one embedder request.++ Text past :data:`EMBED_TEXT_MAX_CHARS` is dropped, and dropping it is+ reported. The cap was sized when the largest page in a corpus was well+ under it, so for a long time it bound on nothing; a corpus whose page mix+ has since changed can push pages over it, and the characters past the cut+ are simply absent from the vector. Nothing downstream can tell that apart+ from a page that never said those words, so the only place the loss is+ observable is here.++ Reported at ``error`` although the run continues, because the CLI pins+ ``repowise.core`` to that level unless ``--verbose`` is passed. Anything+ quieter would be discarded by the very commands that write the index,+ which is a report that exists only in the tests for it.+ """for start in range(0, len(items), EMBED_BATCH_MAX_ITEMS):chunk = items[start : start + EMBED_BATCH_MAX_ITEMS]+ for page_id, text, _meta in chunk:+ if len(text) > EMBED_TEXT_MAX_CHARS:+ logger.error(+ # ``page_id`` is empty for the raw ``embed_texts`` path,+ # which embeds loose strings belonging to no page.+ "embed_text_truncated page_id=%s chars=%d chars_dropped=%d cap=%d",+ page_id,+ len(text),+ len(text) - EMBED_TEXT_MAX_CHARS,+ EMBED_TEXT_MAX_CHARS,+ )yield chunk, [text[:EMBED_TEXT_MAX_CHARS] for _, text, _ in chunk]diff --git a/tests/unit/persistence/test_embed_text_cap.py b/tests/unit/persistence/test_embed_text_cap.pynew file mode 100644index 000000000..d420ec0fb--- /dev/null+++ b/tests/unit/persistence/test_embed_text_cap.py@@ -0,0 +1,146 @@+"""The per-input embedding cap reports what it drops.++``EMBED_TEXT_MAX_CHARS`` silently truncates any text past it. That is the+right thing to do — the embedder rejects an oversized input outright, and one+page must not sink the whole batch it travels in — but until now the drop+left no trace at all. A page whose tail is missing from its vector is+indistinguishable, at every later stage, from a page that never contained+those words: search does not find it, and nothing reports a fault.+"""++from __future__ import annotations++import logging++import pytest++from repowise.core.persistence.vector_store._base import (+ EMBED_TEXT_MAX_CHARS,+ iter_embed_chunks,+)++LOGGER = "repowise.core.persistence.vector_store._base"+++def _drain(items: list[tuple[str, str, dict]]) -> list[str]:+ """Consume the generator and return every capped text it yielded."""+ out: list[str] = []+ for _chunk, capped in iter_embed_chunks(items):+ out.extend(capped)+ return out+++def _reports(caplog) -> list[logging.LogRecord]:+ return [r for r in caplog.records if "embed_text_truncated" in r.getMessage()]+++def test_an_over_cap_text_reports_the_page_and_the_characters_dropped(caplog):+ over_by = 1_234+ text = "x" * (EMBED_TEXT_MAX_CHARS + over_by)++ with caplog.at_level(logging.ERROR, logger=LOGGER):+ capped = _drain([("file_page:a/b.py", text, {})])++ assert capped == ["x" * EMBED_TEXT_MAX_CHARS]++ reports = _reports(caplog)+ assert len(reports) == 1+ message = reports[0].getMessage()+ assert "page_id=file_page:a/b.py" in message+ assert f"chars={EMBED_TEXT_MAX_CHARS + over_by}" in message+ assert f"chars_dropped={over_by}" in message+ assert f"cap={EMBED_TEXT_MAX_CHARS}" in message+++def test_the_report_survives_the_level_the_cli_runs_at(caplog):+ """It has to outrank the filter, or it exists only in this file.++ ``repowise init`` — the command that writes the index, and so the one+ that does the truncating — pins ``repowise.core`` to ``ERROR`` unless+ ``--verbose`` is passed. A warning here would be discarded exactly where+ it matters.+ """+ with caplog.at_level(logging.ERROR, logger=LOGGER):+ _drain([("file_page:a.py", "x" * (EMBED_TEXT_MAX_CHARS + 1), {})])++ assert [r.levelno for r in _reports(caplog)] == [logging.ERROR]+++def test_an_under_cap_text_reports_nothing(caplog):+ with caplog.at_level(logging.ERROR, logger=LOGGER):+ capped = _drain([("file_page:a/b.py", "short body", {})])++ assert capped == ["short body"]+ assert _reports(caplog) == []+++def test_a_text_exactly_at_the_cap_reports_nothing(caplog):+ """The cap is the largest size that survives whole, not the first casualty."""+ text = "x" * EMBED_TEXT_MAX_CHARS++ with caplog.at_level(logging.ERROR, logger=LOGGER):+ capped = _drain([("file_page:a/b.py", text, {})])++ assert capped == [text]+ assert _reports(caplog) == []+++def test_each_over_cap_page_is_named_separately(caplog):+ """A run reports every page it truncated, not that truncation happened.++ The count of affected pages is what sizes the fix; a single "some pages+ were truncated" line cannot be acted on.+ """+ items = [+ ("file_page:one.py", "x" * (EMBED_TEXT_MAX_CHARS + 10), {}),+ ("file_page:two.py", "small", {}),+ ("file_page:three.py", "x" * (EMBED_TEXT_MAX_CHARS + 20), {}),+ ]++ with caplog.at_level(logging.ERROR, logger=LOGGER):+ _drain(items)++ named = [r.getMessage() for r in _reports(caplog)]+ assert len(named) == 2+ assert "page_id=file_page:one.py" in named[0]+ assert "page_id=file_page:three.py" in named[1]+++def test_pages_are_named_across_chunk_boundaries(caplog):+ """Truncation is per item, and the batching must not hide a later chunk.++ ``iter_embed_chunks`` slices into batches; an over-cap page in the second+ slice is exactly as invisible as one in the first, and is reported the+ same way.+ """+ items: list[tuple[str, str, dict]] = [(f"file_page:{i}.py", "small", {}) for i in range(20)]+ items[19] = ("file_page:last.py", "x" * (EMBED_TEXT_MAX_CHARS + 5), {})++ with caplog.at_level(logging.ERROR, logger=LOGGER):+ _drain(items)++ named = [r.getMessage() for r in _reports(caplog)]+ assert len(named) == 1+ assert "page_id=file_page:last.py" in named[0]+++async def test_embed_texts_reports_truncation_too(caplog):+ """The loose-string path shares the cap, so it shares the report.++ It embeds text belonging to no page, so ``page_id`` is empty — an honest+ blank rather than a fabricated id.+ """+ from repowise.core.persistence.vector_store.in_memory import InMemoryVectorStore+ from repowise.core.providers.embedding.base import MockEmbedder++ store = InMemoryVectorStore(MockEmbedder())++ with caplog.at_level(logging.ERROR, logger=LOGGER):+ vectors = await store.embed_texts(["x" * (EMBED_TEXT_MAX_CHARS + 7)])++ assert vectors is not None and len(vectors) == 1+ reports = _reports(caplog)+ assert len(reports) == 1+ assert "page_id= " in reports[0].getMessage() + " "+ assert "chars_dropped=7" in reports[0].getMessage()