Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(core): decode as utf-8 instead of the platform locale (#1466)
Raghav Chamadiya334451ce309e
diff --git a/packages/core/src/repowise/core/analysis/dead_code/dynamic_markers.py b/packages/core/src/repowise/core/analysis/dead_code/dynamic_markers.pyindex 1683c75e8..b3293822f 100644--- a/packages/core/src/repowise/core/analysis/dead_code/dynamic_markers.py+++ b/packages/core/src/repowise/core/analysis/dead_code/dynamic_markers.py@@ -17,7 +17,6 @@ in Phase 2 stays mechanical.from __future__ import annotations-import localefrom pathlib import Pathfrom repowise.core.ids import is_external@@ -35,18 +34,20 @@ def read_source_text(``source_map`` (keyed by the same repo-relative path as ``parsed_files``),so a hit here replaces a full-repo disk pass with a dict lookup.- Decoding is deliberately identical on both paths, and identical to the- ``Path.read_text(errors="ignore")`` these prepasses used to call, which- resolves ``encoding=None`` to the locale encoding at call time. Every- marker is ASCII, but pinning the codec keeps a source_map hit and a disk- fallback from ever disagreeing about the same bytes.+ Decoding is deliberately identical on both paths, so a ``source_map`` hit+ and a disk fallback can never disagree about the same bytes. The codec is+ utf-8, not the locale's: this used to pin ``getpreferredencoding()`` to+ reproduce the ``Path.read_text(errors="ignore")`` the prepasses called,+ but that call was itself the bug: source files are utf-8, and the locale+ codec is cp1252 on a default Windows install. Every marker is ASCII, so no+ marker match changes; the surrounding text stops being mojibake."""data: bytes | None = Noneif source_map is not None:data = source_map.get(rel_path)if data is None:data = Path(abs_path).read_bytes()- return data.decode(locale.getpreferredencoding(False), errors="ignore")+ return data.decode("utf-8", errors="ignore")# Patterns in source that indicate dynamic/runtime imports, keyed by suffix.diff --git a/packages/core/src/repowise/core/analysis/history_scan.py b/packages/core/src/repowise/core/analysis/history_scan.pyindex a6a3b9a8d..2c0977d66 100644--- a/packages/core/src/repowise/core/analysis/history_scan.py+++ b/packages/core/src/repowise/core/analysis/history_scan.py@@ -59,6 +59,13 @@ def _run_git(repo_path: Path, args: list[str], *, timeout: float = 30.0) -> str:cwd=str(repo_path),capture_output=True,text=True,+ # git writes utf-8. ``text=True`` alone decodes with the locale+ # codec, so on a default Windows install a commit subject holding+ # an em dash raised UnicodeDecodeError. That is a ValueError, which+ # the handler below does not catch, so the scan died rather than+ # degrading to "".+ encoding="utf-8",+ errors="replace",timeout=timeout,)except (OSError, subprocess.SubprocessError):diff --git a/packages/core/src/repowise/core/generation/kg_context.py b/packages/core/src/repowise/core/generation/kg_context.pyindex 369384225..7ca59792e 100644--- a/packages/core/src/repowise/core/generation/kg_context.py+++ b/packages/core/src/repowise/core/generation/kg_context.py@@ -73,9 +73,14 @@ class KnowledgeGraphContext:def _load(self, path: Path, repo_root: Path | None = None) -> None:try:- with open(path) as f:- kg = json.load(f)- except (json.JSONDecodeError, OSError) as e:+ # Explicit utf-8: a bare open() decodes with the locale codec,+ # which is cp1252 on a default Windows install. An undefined byte+ # (0x81/0x8D/0x8F/0x90/0x9D) raises UnicodeDecodeError, and since+ # that is a ValueError rather than an OSError it escaped the+ # handler below and took the whole generation run down. Matches+ # analysis/knowledge_graph.py, which loads the same artifact.+ kg = json.loads(path.read_text(encoding="utf-8"))+ except (json.JSONDecodeError, OSError, UnicodeDecodeError) as e:logger.warning("kg_context_load_failed", path=str(path), error=str(e))returndiff --git a/packages/core/src/repowise/core/ingestion/framework_edges/fastapi.py b/packages/core/src/repowise/core/ingestion/framework_edges/fastapi.pyindex f2fdc93e1..0204853f0 100644--- a/packages/core/src/repowise/core/ingestion/framework_edges/fastapi.py+++ b/packages/core/src/repowise/core/ingestion/framework_edges/fastapi.py@@ -6,7 +6,6 @@ Split out of ``framework_edges.py`` (PR 3.5) — behaviour-preserving move.from __future__ import annotationsimport re-from pathlib import Pathfrom typing import TYPE_CHECKING, Anyfrom ..resolvers import ResolverContext, resolve_import@@ -14,6 +13,7 @@ from .base import (DetectionContext,FrameworkHandler,_add_edge_if_new,+ read_text,)if TYPE_CHECKING:@@ -47,10 +47,7 @@ def _add_fastapi_edges(for path, parsed in parsed_files.items():if parsed.file_info.language != "python":continue- try:- source = Path(parsed.file_info.abs_path).read_text(errors="ignore")- except Exception:- continue+ source = read_text(parsed)for match in router_re.finditer(source):var_name = match.group(1)target = var_to_file.get(var_name)diff --git a/packages/core/src/repowise/core/ingestion/framework_edges/flask.py b/packages/core/src/repowise/core/ingestion/framework_edges/flask.pyindex 625314a16..84ec527b0 100644--- a/packages/core/src/repowise/core/ingestion/framework_edges/flask.py+++ b/packages/core/src/repowise/core/ingestion/framework_edges/flask.py@@ -6,7 +6,6 @@ Split out of ``framework_edges.py`` (PR 3.5) — behaviour-preserving move.from __future__ import annotationsimport re-from pathlib import Pathfrom typing import TYPE_CHECKING, Anyfrom ..resolvers import ResolverContext, resolve_import@@ -14,6 +13,7 @@ from .base import (DetectionContext,FrameworkHandler,_add_edge_if_new,+ read_text,)if TYPE_CHECKING:@@ -47,10 +47,7 @@ def _add_flask_edges(for path, parsed in parsed_files.items():if parsed.file_info.language != "python":continue- try:- source = Path(parsed.file_info.abs_path).read_text(errors="ignore")- except Exception:- continue+ source = read_text(parsed)for match in bp_re.finditer(source):var_name = match.group(1)target = var_to_file.get(var_name)diff --git a/packages/core/src/repowise/core/ingestion/resolvers/context.py b/packages/core/src/repowise/core/ingestion/resolvers/context.pyindex 3b913d02f..b7c0c4d14 100644--- a/packages/core/src/repowise/core/ingestion/resolvers/context.py+++ b/packages/core/src/repowise/core/ingestion/resolvers/context.py@@ -105,7 +105,10 @@ class ResolverContext:]:if candidate.exists():try:- with open(candidate) as f:+ # compile_commands.json is generated by the build system+ # and is utf-8; a bare open() would decode it with the+ # locale codec instead.+ with open(candidate, encoding="utf-8") as f:commands = json.load(f)result: dict[str, dict] = {}for entry in commands:diff --git a/packages/server/src/repowise/server/mcp_server/_code_rationale.py b/packages/server/src/repowise/server/mcp_server/_code_rationale.pyindex 22e367b5b..31a302002 100644--- a/packages/server/src/repowise/server/mcp_server/_code_rationale.py+++ b/packages/server/src/repowise/server/mcp_server/_code_rationale.py@@ -404,6 +404,13 @@ def grep_comment_candidates(cwd=str(root),capture_output=True,text=True,+ # Matched source lines come back verbatim, so this is the most+ # encoding-sensitive git output we read. text=True alone decodes+ # with the locale codec; one non-cp1252 byte in any hunk raised+ # UnicodeDecodeError, which the handler below swallows, so get_why+ # returned no rationale anchors at all rather than fewer.+ encoding="utf-8",+ errors="replace",stdin=subprocess.DEVNULL,timeout=_GREP_TIMEOUT_S,)diff --git a/packages/server/src/repowise/server/mcp_server/tool_why.py b/packages/server/src/repowise/server/mcp_server/tool_why.pyindex 76adf400c..51c95e66e 100644--- a/packages/server/src/repowise/server/mcp_server/tool_why.py+++ b/packages/server/src/repowise/server/mcp_server/tool_why.py@@ -886,6 +886,11 @@ async def _run_git_log(cwd=repo_path,capture_output=True,text=True,+ # ``%s`` is the commit subject and ``%an`` the author name, both+ # utf-8 from git. text=True alone decodes with the locale codec,+ # which is cp1252 on a default Windows install.+ encoding="utf-8",+ errors="replace",timeout=10,# See commits_since() in core/precedent/currency.py: a git child# that inherits this server's JSON-RPC stdin can wedge the@@ -921,6 +926,8 @@ async def _run_git_log(cwd=repo_path,capture_output=True,text=True,+ encoding="utf-8", # see above+ errors="replace",timeout=10,stdin=subprocess.DEVNULL, # see above)diff --git a/packages/server/src/repowise/server/services/c4_builder/architecture.py b/packages/server/src/repowise/server/services/c4_builder/architecture.pyindex 1defa9710..0a4b59083 100644--- a/packages/server/src/repowise/server/services/c4_builder/architecture.py+++ b/packages/server/src/repowise/server/services/c4_builder/architecture.py@@ -170,7 +170,10 @@ def _load_knowledge_graph(path: str) -> dict | None:if not path or not os.path.isfile(path):return Nonetry:- with open(path) as f:+ # Explicit utf-8: a bare open() decodes with the locale codec, so on a+ # default Windows install a non-ASCII knowledge graph cost the whole+ # C4 architecture view via the handler below.+ with open(path, encoding="utf-8") as f:data = json.load(f)except (json.JSONDecodeError, OSError, UnicodeDecodeError):logger.warning("kg_file_unreadable path=%s", path, exc_info=True)diff --git a/tests/unit/generation/test_kg_context_encoding.py b/tests/unit/generation/test_kg_context_encoding.pynew file mode 100644index 000000000..e0c289f22--- /dev/null+++ b/tests/unit/generation/test_kg_context_encoding.py@@ -0,0 +1,44 @@+"""``KnowledgeGraphContext`` must not die on a knowledge graph it cannot decode.++``_load`` opened the artifact with a bare ``open()``, which decodes with the+locale codec, cp1252 on a default Windows install. A byte undefined in that+codec (0x81/0x8D/0x8F/0x90/0x9D) raises ``UnicodeDecodeError``, and because+that is a ``ValueError`` rather than an ``OSError`` it escaped the handler and+took the whole generation run down.++Two halves: the file is read as utf-8, and a genuinely undecodable file+degrades to "no KG" instead of raising.+"""++from __future__ import annotations++import json+from pathlib import Path++from repowise.core.generation.kg_context import KnowledgeGraphContext+++def _write_kg(path: Path, nodes: list[dict]) -> None:+ path.write_text(json.dumps({"nodes": nodes}), encoding="utf-8")+++def test_non_ascii_knowledge_graph_loads(tmp_path: Path) -> None:+ """Accented text in a node label is utf-8 on disk and must decode as utf-8,+ whatever the process locale says."""+ kg = tmp_path / ".repowise" / "knowledge-graph.json"+ kg.parent.mkdir(parents=True)+ _write_kg(kg, [{"filePath": "src/app.py", "label": "Ingestión, la búsqueda"}])++ ctx = KnowledgeGraphContext(kg_path=kg)++ assert ctx.available is True+++def test_undecodable_knowledge_graph_degrades(tmp_path: Path) -> None:+ kg = tmp_path / ".repowise" / "knowledge-graph.json"+ kg.parent.mkdir(parents=True)+ kg.write_bytes(b'{"nodes": [{"filePath": "\x81\x90 broken"}]}')++ ctx = KnowledgeGraphContext(kg_path=kg)++ assert ctx.available is Falsediff --git a/tests/unit/generation/test_kg_enrichment.py b/tests/unit/generation/test_kg_enrichment.pyindex 2866c3e78..22cfb6d26 100644--- a/tests/unit/generation/test_kg_enrichment.py+++ b/tests/unit/generation/test_kg_enrichment.py@@ -209,7 +209,10 @@ class TestOnboardingTablesAreRead:return [pathfor path in PACKAGES_ROOT.rglob("*.py")- if path != SLOTS_PATH and name in path.read_text()+ # Explicit utf-8: this sweeps every source file in the repo, and a+ # bare read_text() decodes with the locale codec, so on Windows the+ # first module holding a non-cp1252 byte failed the whole test.+ if path != SLOTS_PATH and name in path.read_text(encoding="utf-8")]def test_every_table_has_a_reader(self):