:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(autoresearch): retry a cut-off sandbox readback and report its stdout (#119142)
posthog[bot]a81c170a5ab0
diff --git a/products/autoresearch/backend/inference/sandbox.py b/products/autoresearch/backend/inference/sandbox.pyindex 928b05238564..e5ab08ac4901 100644--- a/products/autoresearch/backend/inference/sandbox.py+++ b/products/autoresearch/backend/inference/sandbox.py@@ -149,6 +149,7 @@ _FILE_END = "<<<AUTORESEARCH_FILE_END>>>"# store accepts, so a model that passes readback also passes persistence._MAX_READBACK_BYTES = MAX_ARTIFACT_BYTES_MAX_SCRIPT_LOG_BYTES = 4000+_READBACK_EXCERPT_CHARS = 80# Keys train.py must write into output.json._REQUIRED_METRIC_KEYS = ("holdout_auc", "n_train", "n_features")# The worker expands every feature column across every row before the matrix reaches the@@ -1144,10 +1145,7 @@ def _read_file(sandbox: SandboxExecutor, rel_path: str) -> str:write_file is the only input channel and execute→stdout the only output channel,so we cat the file and slice between sentinels to survive any stray shell output."""- result = sandbox.execute(_readback_command(rel_path, encode=False), timeout_seconds=60)- if result.exit_code != 0:- raise SandboxInferenceError(f"reading {rel_path} failed (exit {result.exit_code}): {result.stderr[:500]}")- body = _between_sentinels(result.stdout)+ body = _readback(sandbox, rel_path, encode=False, timeout_seconds=60)if len(body.encode("utf-8")) > _MAX_READBACK_BYTES:raise SandboxInferenceError(f"{rel_path} is over the {_MAX_READBACK_BYTES}-byte readback cap")return body@@ -1155,10 +1153,7 @@ def _read_file(sandbox: SandboxExecutor, rel_path: str) -> str:def _read_binary_file(sandbox: SandboxExecutor, rel_path: str) -> bytes:"""Read a binary file (e.g. model.pkl) the bundle wrote, base64-encoded over the sentinel-bracketed cat."""- result = sandbox.execute(_readback_command(rel_path, encode=True), timeout_seconds=120)- if result.exit_code != 0:- raise SandboxInferenceError(f"reading {rel_path} failed (exit {result.exit_code}): {result.stderr[:500]}")- encoded = "".join(_between_sentinels(result.stdout).split())+ encoded = "".join(_readback(sandbox, rel_path, encode=True, timeout_seconds=120).split())try:content = base64.b64decode(encoded, validate=True)except (binascii.Error, ValueError) as exc:@@ -1170,6 +1165,41 @@ def _read_binary_file(sandbox: SandboxExecutor, rel_path: str) -> bytes:return content+def _readback(sandbox: SandboxExecutor, rel_path: str, *, encode: bool, timeout_seconds: int) -> str:+ # An exit 0 without both sentinels means the provider dropped or cut off stdout. The+ # command only reads a finished file, so one retry is safe.+ command = _readback_command(rel_path, encode=encode)+ result = _execute_readback(sandbox, command, rel_path=rel_path, timeout_seconds=timeout_seconds)+ try:+ return _between_sentinels(result.stdout)+ except SandboxInferenceError:+ logger.warning("autoresearch_readback_retry", rel_path=rel_path, output=_describe_output(result))+ result = _execute_readback(sandbox, command, rel_path=rel_path, timeout_seconds=timeout_seconds)+ try:+ return _between_sentinels(result.stdout)+ except SandboxInferenceError as exc:+ raise SandboxInferenceError(f"{exc} reading {rel_path} after a retry: {_describe_output(result)}") from exc+++def _execute_readback(+ sandbox: SandboxExecutor, command: str, *, rel_path: str, timeout_seconds: int+) -> ExecutionResult:+ result = sandbox.execute(command, timeout_seconds=timeout_seconds)+ if result.exit_code != 0:+ raise SandboxInferenceError(f"reading {rel_path} failed (exit {result.exit_code}): {result.stderr[:500]}")+ return result+++def _describe_output(result: ExecutionResult) -> str:+ # The excerpts are short and show the sentinel ends, enough to tell an empty stdout+ # from one cut off mid-file without copying the file body into the error.+ stdout, n = result.stdout, _READBACK_EXCERPT_CHARS+ return (+ f"stdout_chars={len(stdout)} begin={_FILE_BEGIN in stdout} end={_FILE_END in stdout} "+ f"head={stdout[:n]!r} tail={stdout[-n:]!r} stderr={result.stderr[:500]!r}"+ )++def _read_log_tail(sandbox: SandboxExecutor) -> str:result = sandbox.execute(f"tail -c {_MAX_SCRIPT_LOG_BYTES} {_WORKDIR}/{_SCRIPT_LOG}", timeout_seconds=60)return result.stdout if result.exit_code == 0 else "(script log unavailable)"diff --git a/products/autoresearch/backend/inference/test_sandbox_inference.py b/products/autoresearch/backend/inference/test_sandbox_inference.pyindex decd40ed6702..0d4002aee34e 100644--- a/products/autoresearch/backend/inference/test_sandbox_inference.py+++ b/products/autoresearch/backend/inference/test_sandbox_inference.py@@ -87,6 +87,7 @@ class _FakeSandbox:predict_exit: int = 0,readback_exit: int = 0,write_exit_for: str = "",+ cut_off_readbacks: int = 0,):self.written: dict[str, bytes] = {}self.commands: list[str] = []@@ -97,6 +98,7 @@ class _FakeSandbox:self._predict_exit = predict_exitself._readback_exit = readback_exitself._write_exit_for = write_exit_for+ self._cut_off_readbacks = cut_off_readbacksself.destroyed = Falsedef __enter__(self) -> "_FakeSandbox":@@ -119,6 +121,9 @@ class _FakeSandbox:if _FILE_BEGIN in command:if self._readback_exit:return ExecutionResult(stdout="", stderr="missing", exit_code=self._readback_exit)+ if self._cut_off_readbacks:+ self._cut_off_readbacks -= 1+ return ExecutionResult(stdout=f"{_FILE_BEGIN}\nUEFSMQ", stderr="", exit_code=0)if "base64" in command:raw = self._scores_parquet if "scores.parquet" in command else self._model_bytespayload = base64.b64encode(raw).decode()@@ -376,6 +381,18 @@ class TestFileReadback(SimpleTestCase):with self.assertRaises(SandboxInferenceError):_read_binary_file(fake, "model.pkl")+ def test_read_binary_file_retries_a_cut_off_readback_once(self):+ fake = _FakeSandbox(model_bytes=b"PICKLE", cut_off_readbacks=1)+ assert _read_binary_file(fake, "model.pkl") == b"PICKLE"++ def test_read_binary_file_reports_the_stdout_when_every_readback_is_cut_off(self):+ fake = _FakeSandbox(model_bytes=b"PICKLE", cut_off_readbacks=2)+ with self.assertRaises(SandboxInferenceError) as ctx:+ _read_binary_file(fake, "model.pkl")+ message = str(ctx.exception)+ assert "stdout_chars=36 begin=True end=False" in message+ assert "UEFSMQ" in message+def test_readback_command_gates_on_size_before_emitting(self):command = sandbox_inference._readback_command("model.pkl", encode=True)assert command.index("stat -c %s") < command.index("head -c") < command.index("base64")