Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(core): decode as utf-8 instead of the platform locale (#1466)

Raghav Chamadiya334451ce309e
diff --git a/packages/core/src/repowise/core/analysis/dead_code/dynamic_markers.py b/packages/core/src/repowise/core/analysis/dead_code/dynamic_markers.py
index 1683c75e8..b3293822f 100644
--- a/packages/core/src/repowise/core/analysis/dead_code/dynamic_markers.py
+++ b/packages/core/src/repowise/core/analysis/dead_code/dynamic_markers.py
@@ -17,7 +17,6 @@ in Phase 2 stays mechanical.
from __future__ import annotations
-import locale
from pathlib import Path
from repowise.core.ids import is_external
@@ -35,18 +34,20 @@ def read_source_text(
``source_map`` (keyed by the same repo-relative path as ``parsed_files``),
so a hit here replaces a full-repo disk pass with a dict lookup.
- Decoding is deliberately identical on both paths, and identical to the
- ``Path.read_text(errors="ignore")`` these prepasses used to call, which
- resolves ``encoding=None`` to the locale encoding at call time. Every
- marker is ASCII, but pinning the codec keeps a source_map hit and a disk
- fallback from ever disagreeing about the same bytes.
+ Decoding is deliberately identical on both paths, so a ``source_map`` hit
+ and a disk fallback can never disagree about the same bytes. The codec is
+ utf-8, not the locale's: this used to pin ``getpreferredencoding()`` to
+ reproduce the ``Path.read_text(errors="ignore")`` the prepasses called,
+ but that call was itself the bug: source files are utf-8, and the locale
+ codec is cp1252 on a default Windows install. Every marker is ASCII, so no
+ marker match changes; the surrounding text stops being mojibake.
"""
data: bytes | None = None
if source_map is not None:
data = source_map.get(rel_path)
if data is None:
data = Path(abs_path).read_bytes()
- return data.decode(locale.getpreferredencoding(False), errors="ignore")
+ return data.decode("utf-8", errors="ignore")
# Patterns in source that indicate dynamic/runtime imports, keyed by suffix.
diff --git a/packages/core/src/repowise/core/analysis/history_scan.py b/packages/core/src/repowise/core/analysis/history_scan.py
index a6a3b9a8d..2c0977d66 100644
--- a/packages/core/src/repowise/core/analysis/history_scan.py
+++ b/packages/core/src/repowise/core/analysis/history_scan.py
@@ -59,6 +59,13 @@ def _run_git(repo_path: Path, args: list[str], *, timeout: float = 30.0) -> str:
cwd=str(repo_path),
capture_output=True,
text=True,
+ # git writes utf-8. ``text=True`` alone decodes with the locale
+ # codec, so on a default Windows install a commit subject holding
+ # an em dash raised UnicodeDecodeError. That is a ValueError, which
+ # the handler below does not catch, so the scan died rather than
+ # degrading to "".
+ encoding="utf-8",
+ errors="replace",
timeout=timeout,
)
except (OSError, subprocess.SubprocessError):
diff --git a/packages/core/src/repowise/core/generation/kg_context.py b/packages/core/src/repowise/core/generation/kg_context.py
index 369384225..7ca59792e 100644
--- a/packages/core/src/repowise/core/generation/kg_context.py
+++ b/packages/core/src/repowise/core/generation/kg_context.py
@@ -73,9 +73,14 @@ class KnowledgeGraphContext:
def _load(self, path: Path, repo_root: Path | None = None) -> None:
try:
- with open(path) as f:
- kg = json.load(f)
- except (json.JSONDecodeError, OSError) as e:
+ # Explicit utf-8: a bare open() decodes with the locale codec,
+ # which is cp1252 on a default Windows install. An undefined byte
+ # (0x81/0x8D/0x8F/0x90/0x9D) raises UnicodeDecodeError, and since
+ # that is a ValueError rather than an OSError it escaped the
+ # handler below and took the whole generation run down. Matches
+ # analysis/knowledge_graph.py, which loads the same artifact.
+ kg = json.loads(path.read_text(encoding="utf-8"))
+ except (json.JSONDecodeError, OSError, UnicodeDecodeError) as e:
logger.warning("kg_context_load_failed", path=str(path), error=str(e))
return
diff --git a/packages/core/src/repowise/core/ingestion/framework_edges/fastapi.py b/packages/core/src/repowise/core/ingestion/framework_edges/fastapi.py
index f2fdc93e1..0204853f0 100644
--- a/packages/core/src/repowise/core/ingestion/framework_edges/fastapi.py
+++ b/packages/core/src/repowise/core/ingestion/framework_edges/fastapi.py
@@ -6,7 +6,6 @@ Split out of ``framework_edges.py`` (PR 3.5) — behaviour-preserving move.
from __future__ import annotations
import re
-from pathlib import Path
from typing import TYPE_CHECKING, Any
from ..resolvers import ResolverContext, resolve_import
@@ -14,6 +13,7 @@ from .base import (
DetectionContext,
FrameworkHandler,
_add_edge_if_new,
+ read_text,
)
if TYPE_CHECKING:
@@ -47,10 +47,7 @@ def _add_fastapi_edges(
for path, parsed in parsed_files.items():
if parsed.file_info.language != "python":
continue
- try:
- source = Path(parsed.file_info.abs_path).read_text(errors="ignore")
- except Exception:
- continue
+ source = read_text(parsed)
for match in router_re.finditer(source):
var_name = match.group(1)
target = var_to_file.get(var_name)
diff --git a/packages/core/src/repowise/core/ingestion/framework_edges/flask.py b/packages/core/src/repowise/core/ingestion/framework_edges/flask.py
index 625314a16..84ec527b0 100644
--- a/packages/core/src/repowise/core/ingestion/framework_edges/flask.py
+++ b/packages/core/src/repowise/core/ingestion/framework_edges/flask.py
@@ -6,7 +6,6 @@ Split out of ``framework_edges.py`` (PR 3.5) — behaviour-preserving move.
from __future__ import annotations
import re
-from pathlib import Path
from typing import TYPE_CHECKING, Any
from ..resolvers import ResolverContext, resolve_import
@@ -14,6 +13,7 @@ from .base import (
DetectionContext,
FrameworkHandler,
_add_edge_if_new,
+ read_text,
)
if TYPE_CHECKING:
@@ -47,10 +47,7 @@ def _add_flask_edges(
for path, parsed in parsed_files.items():
if parsed.file_info.language != "python":
continue
- try:
- source = Path(parsed.file_info.abs_path).read_text(errors="ignore")
- except Exception:
- continue
+ source = read_text(parsed)
for match in bp_re.finditer(source):
var_name = match.group(1)
target = var_to_file.get(var_name)
diff --git a/packages/core/src/repowise/core/ingestion/resolvers/context.py b/packages/core/src/repowise/core/ingestion/resolvers/context.py
index 3b913d02f..b7c0c4d14 100644
--- a/packages/core/src/repowise/core/ingestion/resolvers/context.py
+++ b/packages/core/src/repowise/core/ingestion/resolvers/context.py
@@ -105,7 +105,10 @@ class ResolverContext:
]:
if candidate.exists():
try:
- with open(candidate) as f:
+ # compile_commands.json is generated by the build system
+ # and is utf-8; a bare open() would decode it with the
+ # locale codec instead.
+ with open(candidate, encoding="utf-8") as f:
commands = json.load(f)
result: dict[str, dict] = {}
for entry in commands:
diff --git a/packages/server/src/repowise/server/mcp_server/_code_rationale.py b/packages/server/src/repowise/server/mcp_server/_code_rationale.py
index 22e367b5b..31a302002 100644
--- a/packages/server/src/repowise/server/mcp_server/_code_rationale.py
+++ b/packages/server/src/repowise/server/mcp_server/_code_rationale.py
@@ -404,6 +404,13 @@ def grep_comment_candidates(
cwd=str(root),
capture_output=True,
text=True,
+ # Matched source lines come back verbatim, so this is the most
+ # encoding-sensitive git output we read. text=True alone decodes
+ # with the locale codec; one non-cp1252 byte in any hunk raised
+ # UnicodeDecodeError, which the handler below swallows, so get_why
+ # returned no rationale anchors at all rather than fewer.
+ encoding="utf-8",
+ errors="replace",
stdin=subprocess.DEVNULL,
timeout=_GREP_TIMEOUT_S,
)
diff --git a/packages/server/src/repowise/server/mcp_server/tool_why.py b/packages/server/src/repowise/server/mcp_server/tool_why.py
index 76adf400c..51c95e66e 100644
--- a/packages/server/src/repowise/server/mcp_server/tool_why.py
+++ b/packages/server/src/repowise/server/mcp_server/tool_why.py
@@ -886,6 +886,11 @@ async def _run_git_log(
cwd=repo_path,
capture_output=True,
text=True,
+ # ``%s`` is the commit subject and ``%an`` the author name, both
+ # utf-8 from git. text=True alone decodes with the locale codec,
+ # which is cp1252 on a default Windows install.
+ encoding="utf-8",
+ errors="replace",
timeout=10,
# See commits_since() in core/precedent/currency.py: a git child
# that inherits this server's JSON-RPC stdin can wedge the
@@ -921,6 +926,8 @@ async def _run_git_log(
cwd=repo_path,
capture_output=True,
text=True,
+ encoding="utf-8", # see above
+ errors="replace",
timeout=10,
stdin=subprocess.DEVNULL, # see above
)
diff --git a/packages/server/src/repowise/server/services/c4_builder/architecture.py b/packages/server/src/repowise/server/services/c4_builder/architecture.py
index 1defa9710..0a4b59083 100644
--- a/packages/server/src/repowise/server/services/c4_builder/architecture.py
+++ b/packages/server/src/repowise/server/services/c4_builder/architecture.py
@@ -170,7 +170,10 @@ def _load_knowledge_graph(path: str) -> dict | None:
if not path or not os.path.isfile(path):
return None
try:
- with open(path) as f:
+ # Explicit utf-8: a bare open() decodes with the locale codec, so on a
+ # default Windows install a non-ASCII knowledge graph cost the whole
+ # C4 architecture view via the handler below.
+ with open(path, encoding="utf-8") as f:
data = json.load(f)
except (json.JSONDecodeError, OSError, UnicodeDecodeError):
logger.warning("kg_file_unreadable path=%s", path, exc_info=True)
diff --git a/tests/unit/generation/test_kg_context_encoding.py b/tests/unit/generation/test_kg_context_encoding.py
new file mode 100644
index 000000000..e0c289f22
--- /dev/null
+++ b/tests/unit/generation/test_kg_context_encoding.py
@@ -0,0 +1,44 @@
+"""``KnowledgeGraphContext`` must not die on a knowledge graph it cannot decode.
+
+``_load`` opened the artifact with a bare ``open()``, which decodes with the
+locale codec, cp1252 on a default Windows install. A byte undefined in that
+codec (0x81/0x8D/0x8F/0x90/0x9D) raises ``UnicodeDecodeError``, and because
+that is a ``ValueError`` rather than an ``OSError`` it escaped the handler and
+took the whole generation run down.
+
+Two halves: the file is read as utf-8, and a genuinely undecodable file
+degrades to "no KG" instead of raising.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from repowise.core.generation.kg_context import KnowledgeGraphContext
+
+
+def _write_kg(path: Path, nodes: list[dict]) -> None:
+ path.write_text(json.dumps({"nodes": nodes}), encoding="utf-8")
+
+
+def test_non_ascii_knowledge_graph_loads(tmp_path: Path) -> None:
+ """Accented text in a node label is utf-8 on disk and must decode as utf-8,
+ whatever the process locale says."""
+ kg = tmp_path / ".repowise" / "knowledge-graph.json"
+ kg.parent.mkdir(parents=True)
+ _write_kg(kg, [{"filePath": "src/app.py", "label": "Ingestión, la búsqueda"}])
+
+ ctx = KnowledgeGraphContext(kg_path=kg)
+
+ assert ctx.available is True
+
+
+def test_undecodable_knowledge_graph_degrades(tmp_path: Path) -> None:
+ kg = tmp_path / ".repowise" / "knowledge-graph.json"
+ kg.parent.mkdir(parents=True)
+ kg.write_bytes(b'{"nodes": [{"filePath": "\x81\x90 broken"}]}')
+
+ ctx = KnowledgeGraphContext(kg_path=kg)
+
+ assert ctx.available is False
diff --git a/tests/unit/generation/test_kg_enrichment.py b/tests/unit/generation/test_kg_enrichment.py
index 2866c3e78..22cfb6d26 100644
--- a/tests/unit/generation/test_kg_enrichment.py
+++ b/tests/unit/generation/test_kg_enrichment.py
@@ -209,7 +209,10 @@ class TestOnboardingTablesAreRead:
return [
path
for path in PACKAGES_ROOT.rglob("*.py")
- if path != SLOTS_PATH and name in path.read_text()
+ # Explicit utf-8: this sweeps every source file in the repo, and a
+ # bare read_text() decodes with the locale codec, so on Windows the
+ # first module holding a non-cp1252 byte failed the whole test.
+ if path != SLOTS_PATH and name in path.read_text(encoding="utf-8")
]
def test_every_table_has_a_reader(self):