Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(autoresearch): retry a cut-off sandbox readback and report its stdout (#119142)

posthog[bot]a81c170a5ab0
diff --git a/products/autoresearch/backend/inference/sandbox.py b/products/autoresearch/backend/inference/sandbox.py
index 928b05238564..e5ab08ac4901 100644
--- a/products/autoresearch/backend/inference/sandbox.py
+++ b/products/autoresearch/backend/inference/sandbox.py
@@ -149,6 +149,7 @@ _FILE_END = "<<<AUTORESEARCH_FILE_END>>>"
# store accepts, so a model that passes readback also passes persistence.
_MAX_READBACK_BYTES = MAX_ARTIFACT_BYTES
_MAX_SCRIPT_LOG_BYTES = 4000
+_READBACK_EXCERPT_CHARS = 80
# Keys train.py must write into output.json.
_REQUIRED_METRIC_KEYS = ("holdout_auc", "n_train", "n_features")
# The worker expands every feature column across every row before the matrix reaches the
@@ -1144,10 +1145,7 @@ def _read_file(sandbox: SandboxExecutor, rel_path: str) -> str:
write_file is the only input channel and execute→stdout the only output channel,
so we cat the file and slice between sentinels to survive any stray shell output.
"""
- result = sandbox.execute(_readback_command(rel_path, encode=False), timeout_seconds=60)
- if result.exit_code != 0:
- raise SandboxInferenceError(f"reading {rel_path} failed (exit {result.exit_code}): {result.stderr[:500]}")
- body = _between_sentinels(result.stdout)
+ body = _readback(sandbox, rel_path, encode=False, timeout_seconds=60)
if len(body.encode("utf-8")) > _MAX_READBACK_BYTES:
raise SandboxInferenceError(f"{rel_path} is over the {_MAX_READBACK_BYTES}-byte readback cap")
return body
@@ -1155,10 +1153,7 @@ def _read_file(sandbox: SandboxExecutor, rel_path: str) -> str:
def _read_binary_file(sandbox: SandboxExecutor, rel_path: str) -> bytes:
"""Read a binary file (e.g. model.pkl) the bundle wrote, base64-encoded over the sentinel-bracketed cat."""
- result = sandbox.execute(_readback_command(rel_path, encode=True), timeout_seconds=120)
- if result.exit_code != 0:
- raise SandboxInferenceError(f"reading {rel_path} failed (exit {result.exit_code}): {result.stderr[:500]}")
- encoded = "".join(_between_sentinels(result.stdout).split())
+ encoded = "".join(_readback(sandbox, rel_path, encode=True, timeout_seconds=120).split())
try:
content = base64.b64decode(encoded, validate=True)
except (binascii.Error, ValueError) as exc:
@@ -1170,6 +1165,41 @@ def _read_binary_file(sandbox: SandboxExecutor, rel_path: str) -> bytes:
return content
+def _readback(sandbox: SandboxExecutor, rel_path: str, *, encode: bool, timeout_seconds: int) -> str:
+ # An exit 0 without both sentinels means the provider dropped or cut off stdout. The
+ # command only reads a finished file, so one retry is safe.
+ command = _readback_command(rel_path, encode=encode)
+ result = _execute_readback(sandbox, command, rel_path=rel_path, timeout_seconds=timeout_seconds)
+ try:
+ return _between_sentinels(result.stdout)
+ except SandboxInferenceError:
+ logger.warning("autoresearch_readback_retry", rel_path=rel_path, output=_describe_output(result))
+ result = _execute_readback(sandbox, command, rel_path=rel_path, timeout_seconds=timeout_seconds)
+ try:
+ return _between_sentinels(result.stdout)
+ except SandboxInferenceError as exc:
+ raise SandboxInferenceError(f"{exc} reading {rel_path} after a retry: {_describe_output(result)}") from exc
+
+
+def _execute_readback(
+ sandbox: SandboxExecutor, command: str, *, rel_path: str, timeout_seconds: int
+) -> ExecutionResult:
+ result = sandbox.execute(command, timeout_seconds=timeout_seconds)
+ if result.exit_code != 0:
+ raise SandboxInferenceError(f"reading {rel_path} failed (exit {result.exit_code}): {result.stderr[:500]}")
+ return result
+
+
+def _describe_output(result: ExecutionResult) -> str:
+ # The excerpts are short and show the sentinel ends, enough to tell an empty stdout
+ # from one cut off mid-file without copying the file body into the error.
+ stdout, n = result.stdout, _READBACK_EXCERPT_CHARS
+ return (
+ f"stdout_chars={len(stdout)} begin={_FILE_BEGIN in stdout} end={_FILE_END in stdout} "
+ f"head={stdout[:n]!r} tail={stdout[-n:]!r} stderr={result.stderr[:500]!r}"
+ )
+
+
def _read_log_tail(sandbox: SandboxExecutor) -> str:
result = sandbox.execute(f"tail -c {_MAX_SCRIPT_LOG_BYTES} {_WORKDIR}/{_SCRIPT_LOG}", timeout_seconds=60)
return result.stdout if result.exit_code == 0 else "(script log unavailable)"
diff --git a/products/autoresearch/backend/inference/test_sandbox_inference.py b/products/autoresearch/backend/inference/test_sandbox_inference.py
index decd40ed6702..0d4002aee34e 100644
--- a/products/autoresearch/backend/inference/test_sandbox_inference.py
+++ b/products/autoresearch/backend/inference/test_sandbox_inference.py
@@ -87,6 +87,7 @@ class _FakeSandbox:
predict_exit: int = 0,
readback_exit: int = 0,
write_exit_for: str = "",
+ cut_off_readbacks: int = 0,
):
self.written: dict[str, bytes] = {}
self.commands: list[str] = []
@@ -97,6 +98,7 @@ class _FakeSandbox:
self._predict_exit = predict_exit
self._readback_exit = readback_exit
self._write_exit_for = write_exit_for
+ self._cut_off_readbacks = cut_off_readbacks
self.destroyed = False
def __enter__(self) -> "_FakeSandbox":
@@ -119,6 +121,9 @@ class _FakeSandbox:
if _FILE_BEGIN in command:
if self._readback_exit:
return ExecutionResult(stdout="", stderr="missing", exit_code=self._readback_exit)
+ if self._cut_off_readbacks:
+ self._cut_off_readbacks -= 1
+ return ExecutionResult(stdout=f"{_FILE_BEGIN}\nUEFSMQ", stderr="", exit_code=0)
if "base64" in command:
raw = self._scores_parquet if "scores.parquet" in command else self._model_bytes
payload = base64.b64encode(raw).decode()
@@ -376,6 +381,18 @@ class TestFileReadback(SimpleTestCase):
with self.assertRaises(SandboxInferenceError):
_read_binary_file(fake, "model.pkl")
+ def test_read_binary_file_retries_a_cut_off_readback_once(self):
+ fake = _FakeSandbox(model_bytes=b"PICKLE", cut_off_readbacks=1)
+ assert _read_binary_file(fake, "model.pkl") == b"PICKLE"
+
+ def test_read_binary_file_reports_the_stdout_when_every_readback_is_cut_off(self):
+ fake = _FakeSandbox(model_bytes=b"PICKLE", cut_off_readbacks=2)
+ with self.assertRaises(SandboxInferenceError) as ctx:
+ _read_binary_file(fake, "model.pkl")
+ message = str(ctx.exception)
+ assert "stdout_chars=36 begin=True end=False" in message
+ assert "UEFSMQ" in message
+
def test_readback_command_gates_on_size_before_emitting(self):
command = sandbox_inference._readback_command("model.pkl", encode=True)
assert command.index("stat -c %s") < command.index("head -c") < command.index("base64")