Graph-Native Infrastructure for Context and Accountable AI Systems
Python agent-memoryaiai-governanceai-infrastructureartificial-intelligencecontext-engineeringcontext-graphsdata-engineeringdecision-intelligencedeveloper-toolsexplainable-aigenerative-aigraph-ragknowledge-graphllmontologyprovenancepythonreasoningsemantic-search
fix(utils): write console progress to stderr instead of stdout
dex0shubham70dfbf151c80
diff --git a/semantica/utils/progress_tracker.py b/semantica/utils/progress_tracker.pyindex f27768d4..4798b4b9 100644--- a/semantica/utils/progress_tracker.py+++ b/semantica/utils/progress_tracker.py@@ -43,7 +43,7 @@ from contextlib import contextmanagerfrom dataclasses import dataclass, fieldfrom datetime import datetimefrom pathlib import Path-from typing import Any, Callable, Dict, List, Optional, Tuple, Union+from typing import Any, Callable, Dict, List, Optional, TextIO, Tuple, Unionfrom .logging import get_logger@@ -144,14 +144,37 @@ class ProgressDisplay(ABC):class ConsoleProgressDisplay(ProgressDisplay):"""Console progress display with real-time updates."""- def __init__(self, use_emoji: bool = True, update_interval: float = 0.1):+ def __init__(+ self,+ use_emoji: bool = True,+ update_interval: float = 0.1,+ stream: Optional[TextIO] = None,+ ):+ """Initialize the console display.++ Args:+ use_emoji: Whether to decorate output with emoji.+ update_interval: Minimum seconds between redraws.+ stream: Where progress is written. Defaults to ``sys.stderr``.++ Progress is diagnostic output, so stderr is the correct stream+ for it, and stdout must stay clean for programs that carry a+ machine-readable protocol on it β the stdio MCP servers put+ newline-delimited JSON-RPC there, and a progress bar on stdout+ corrupts that stream.++ Left as ``None``, the stream is resolved on each write rather+ than captured here, so a later rebinding of ``sys.stderr``+ (pytest capture, for instance) is honoured.+ """self.use_emoji = use_emoji-- # Check if stdout supports emojis (especially on Windows)+ self._stream = stream++ # Check if the target stream supports emojis (especially on Windows)if self.use_emoji:try:- # Try encoding a test emoji with stdout's encoding- encoding = getattr(sys.stdout, "encoding", None)+ # Try encoding a test emoji with the stream's encoding+ encoding = getattr(self.stream, "encoding", None)if encoding:"π§ ".encode(encoding)except (UnicodeEncodeError, LookupError, AttributeError):@@ -162,6 +185,11 @@ class ConsoleProgressDisplay(ProgressDisplay):self.current_lines: Dict[str, str] = {}self.lock = threading.Lock()+ @property+ def stream(self) -> TextIO:+ """The stream progress is written to; ``sys.stderr`` unless overridden."""+ return self._stream if self._stream is not None else sys.stderr+def _should_update(self) -> bool:"""Check if enough time has passed for update."""now = time.time()@@ -259,15 +287,16 @@ class ConsoleProgressDisplay(ProgressDisplay):return f"{base_msg}: {message}"def _safe_write(self, text: str) -> None:- """Safely write text to stdout handling encoding errors."""+ """Safely write text to the progress stream handling encoding errors."""+ stream = self.streamtry:- sys.stdout.write(text)+ stream.write(text)except UnicodeEncodeError:# Fallback: encode with replacement and write decoded# Use ascii as safe fallback if encoding is unknown or caused error- encoding = getattr(sys.stdout, "encoding", None) or "ascii"+ encoding = getattr(stream, "encoding", None) or "ascii"safe_text = text.encode(encoding, errors="replace").decode(encoding)- sys.stdout.write(safe_text)+ stream.write(safe_text)def update(self, item: ProgressItem) -> None:"""Update console progress display."""@@ -302,11 +331,11 @@ class ConsoleProgressDisplay(ProgressDisplay):self._display_item_line(pipeline_item)self._safe_write("\n")- sys.stdout.flush()+ self.stream.flush()else:# Original single-item displayself._display_item_line(item)- sys.stdout.flush()+ self.stream.flush()def _display_item_line(self, item: ProgressItem) -> None:"""Display a single progress item line."""@@ -468,13 +497,13 @@ class ConsoleProgressDisplay(ProgressDisplay):f"Completed: {completed} | Failed: {failed} | Total Time: {total_time:.2f}s\n")self._safe_write("=" * 80 + "\n")- sys.stdout.flush()+ self.stream.flush()def clear(self) -> None:"""Clear console display."""with self.lock:self._safe_write("\r" + " " * 100 + "\r")- sys.stdout.flush()+ self.stream.flush()self.current_lines.clear()diff --git a/tests/deduplication/test_deduplication.py b/tests/deduplication/test_deduplication.pyindex 8dc7b7c2..54402a46 100644--- a/tests/deduplication/test_deduplication.py+++ b/tests/deduplication/test_deduplication.py@@ -291,8 +291,8 @@ class TestDeduplication(unittest.TestCase):class TestProgressTrackerEncoding(unittest.TestCase):"""Regression tests for issue #531 β Unicode crash on cp1252 Windows consoles."""- def _make_cp1252_stdout(self):- """Return a stdout-like object that raises UnicodeEncodeError for non-cp1252 chars."""+ def _make_cp1252_stream(self):+ """Return a stream that raises UnicodeEncodeError for non-cp1252 chars."""class CP1252Writer:encoding = "cp1252"def write(self, text):@@ -304,39 +304,39 @@ class TestProgressTrackerEncoding(unittest.TestCase):def test_safe_write_does_not_crash_on_cp1252(self):"""_safe_write must not raise UnicodeEncodeError on a cp1252 console."""display = ConsoleProgressDisplay()- orig = sys.stdout- sys.stdout = self._make_cp1252_stdout()+ orig = sys.stderr+ sys.stderr = self._make_cp1252_stream()try:display._safe_write("π§ Semantica - π Current Progress\n")except UnicodeEncodeError:- self.fail("_safe_write raised UnicodeEncodeError on cp1252 stdout")+ self.fail("_safe_write raised UnicodeEncodeError on a cp1252 stream")finally:- sys.stdout = orig+ sys.stderr = origdef test_update_pipeline_header_does_not_crash_on_cp1252(self):"""update() pipeline header write must not crash on a cp1252 console (issue #531)."""from semantica.utils.progress_tracker import ProgressItemdisplay = ConsoleProgressDisplay()display.use_emoji = True # force emoji path to exercise the fixed branch- orig = sys.stdout- sys.stdout = self._make_cp1252_stdout()+ orig = sys.stderr+ sys.stderr = self._make_cp1252_stream()try:display._safe_write("π§ Semantica - π Current Progress\n")display._safe_write("=" * 150 + "\n")except UnicodeEncodeError:self.fail("Pipeline header write raised UnicodeEncodeError on cp1252 stdout")finally:- sys.stdout = orig+ sys.stderr = origdef test_emoji_detection_disables_on_cp1252(self):- """ConsoleProgressDisplay should auto-disable emoji when stdout is cp1252."""- orig = sys.stdout- sys.stdout = self._make_cp1252_stdout()+ """ConsoleProgressDisplay should auto-disable emoji when the progress stream is cp1252."""+ orig = sys.stderr+ sys.stderr = self._make_cp1252_stream()try:display = ConsoleProgressDisplay()- self.assertFalse(display.use_emoji, "use_emoji should be False on cp1252 stdout")+ self.assertFalse(display.use_emoji, "use_emoji should be False on a cp1252 progress stream")finally:- sys.stdout = orig+ sys.stderr = origclass TestResultLimiting(unittest.TestCase):diff --git a/tests/utils/test_progress_stream.py b/tests/utils/test_progress_stream.pynew file mode 100644index 00000000..0e851710--- /dev/null+++ b/tests/utils/test_progress_stream.py@@ -0,0 +1,160 @@+"""Console progress must be written to stderr, never to stdout.++Progress is diagnostic output. Writing it to stdout corrupts any program that+carries a machine-readable protocol there β the stdio MCP servers put+newline-delimited JSON-RPC on stdout, and a progress bar interleaved with a+response body makes that response unparseable (#1134).++Both servers currently defend against this by setting+SEMANTICA_DISABLE_PROGRESS, and a console display is only attached when the+console is interactive. Those are containments, not the fix: they give up+progress output entirely, and every future entry point has to remember them.+Writing to the correct stream in the first place is what these tests pin.+"""++import importlib+import io+import sys++import pytest+++def progress_module(monkeypatch):+ """Import the real progress_tracker, bypassing a mocked sys.modules entry.++ tests/test_extractors_dispatch.py assigns a MagicMock over+ 'semantica.utils.progress_tracker' at import time and never restores it, so+ a plain module-level import here returns mocks when that file has already+ run. Dropping the cached entry via monkeypatch re-imports the real module+ and restores whatever was there afterwards.+ """+ name = "semantica.utils.progress_tracker"+ monkeypatch.delitem(sys.modules, name, raising=False)+ module = importlib.import_module(name)+ assert hasattr(module, "__file__"), "expected the real module, got a stand-in"+ return module+++def display_cls(progress_module):+ return progress_module.ConsoleProgressDisplay+++def make_item(progress_module):+ def _make(**overrides):+ defaults = dict(+ module="kg",+ submodule="Reasoner",+ message="Inferring facts",+ status="running",+ total_items=10,+ processed_items=3,+ )+ defaults.update(overrides)+ return progress_module.ProgressItem(**defaults)++ return _make+++class TestProgressStreamDefaults:+ def test_defaults_to_stderr(self, display_cls):+ assert display_cls().stream is sys.stderr++ def test_default_is_not_stdout(self, display_cls):+ """The whole point: stdout stays clean for protocol traffic."""+ assert display_cls().stream is not sys.stdout++ def test_stream_follows_rebinding(self, display_cls, monkeypatch):+ """Resolved per write, so pytest capture and later rebinds are honoured."""+ display = display_cls()+ replacement = io.StringIO()+ monkeypatch.setattr(sys, "stderr", replacement)+ assert display.stream is replacement++ def test_explicit_stream_overrides_the_default(self, display_cls):+ buffer = io.StringIO()+ assert display_cls(stream=buffer).stream is buffer+++class TestProgressWritesGoToTheStream:+ def test_update_writes_to_the_configured_stream(self, display_cls, make_item):+ buffer = io.StringIO()+ display = display_cls(stream=buffer, use_emoji=False, update_interval=0.0)++ display.update(make_item())++ assert buffer.getvalue(), "progress should have been rendered"++ def test_update_writes_nothing_to_stdout(self, display_cls, make_item, monkeypatch):+ """Regression guard for #1134: stdout must stay untouched."""+ fake_stdout = io.StringIO()+ monkeypatch.setattr(sys, "stdout", fake_stdout)+ buffer = io.StringIO()++ display = display_cls(stream=buffer, use_emoji=False, update_interval=0.0)+ display.update(make_item())+ display.clear()++ assert fake_stdout.getvalue() == "", (+ f"console progress leaked to stdout: {fake_stdout.getvalue()!r}"+ )++ def test_default_display_writes_nothing_to_stdout(+ self, display_cls, make_item, monkeypatch+ ):+ """Same guard without an explicit stream, i.e. the real default path."""+ fake_stdout = io.StringIO()+ fake_stderr = io.StringIO()+ monkeypatch.setattr(sys, "stdout", fake_stdout)+ monkeypatch.setattr(sys, "stderr", fake_stderr)++ display = display_cls(use_emoji=False, update_interval=0.0)+ display.update(make_item())++ assert fake_stdout.getvalue() == ""+ assert fake_stderr.getvalue(), "progress should have gone to stderr"++ def test_clear_flushes_the_stream_not_stdout(self, display_cls, monkeypatch):+ flushed = []++ class RecordingStream(io.StringIO):+ def flush(self):+ flushed.append("stream")++ class ExplodingStdout(io.StringIO):+ def flush(self): # pragma: no cover - fails the test if reached+ raise AssertionError("progress must not flush stdout")++ monkeypatch.setattr(sys, "stdout", ExplodingStdout())+ display = display_cls(+ stream=RecordingStream(), use_emoji=False, update_interval=0.0+ )++ display.clear()++ assert flushed == ["stream"]+++class TestEncodingFallback:+ def test_falls_back_when_the_stream_cannot_encode(self, display_cls):+ """A cp1252-style console must not raise on emoji; it degrades instead."""++ class AsciiOnly(io.StringIO):+ encoding = "ascii"++ def write(self, text):+ text.encode("ascii") # raises UnicodeEncodeError on emoji+ return super().write(text)++ buffer = AsciiOnly()+ display = display_cls(stream=buffer, use_emoji=True, update_interval=0.0)++ display._safe_write("progress \U0001f504 bar\n")++ assert "progress" in buffer.getvalue()+++if __name__ == "__main__":+ pytest.main([__file__])