Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

fix(serve): /v1/tools/python requires OS-level isolation, as /v1/tools/bash does (#2166)

Alpamys Makazhan080af6fa0738
diff --git a/changelog.d/0.76.0/2166.fixed.md b/changelog.d/0.76.0/2166.fixed.md
new file mode 100644
index 00000000..968fbdce
--- /dev/null
+++ b/changelog.d/0.76.0/2166.fixed.md
@@ -0,0 +1,6 @@
+- `soup serve`: `POST /v1/tools/python` now answers HTTP 501 without running the code where
+ the host has no OS-level isolation (Windows, Linux with Python 3.11) and where the Linux
+ namespaces cannot be created for the run (a kernel or container that refuses them), the
+ same as `POST /v1/tools/bash`. It runs where `os.unshare` creates the namespaces (Linux,
+ Python 3.12+) or `sandbox-exec` is available (macOS). `code_exec_reward` and the recipe
+ `code` node are unchanged (#2166).
diff --git a/docs/serving-and-export.md b/docs/serving-and-export.md
index 50df0b94..0c216050 100644
--- a/docs/serving-and-export.md
+++ b/docs/serving-and-export.md
@@ -985,7 +985,11 @@ Three POST routes are now available on `soup serve`:
- **`POST /v1/tools/python`** — runs caller-supplied Python code in an isolated
sub-process. Bounded by 512 MB memory, 5-second wall-clock timeout, and a
- single-worker concurrency cap.
+ cap of four concurrent runs (shared with the code-execution rewards). Like `/v1/tools/bash`, it runs the code only under
+ OS-level isolation: on Linux with Python 3.12+ when `os.unshare` creates new user,
+ network and PID namespaces for the run, and on macOS when `sandbox-exec` is on `PATH`.
+ Anywhere else it answers HTTP 501 without running the code, including on Windows, on
+ Linux with Python 3.11, and on a Linux host or container that refuses those namespaces.
- **`POST /v1/tools/web_search`** — searches the web and returns domain-filtered results
as `[{url, title, snippet}]` with snippets sanitized (null bytes stripped).
Deny-by-default via `WebSearchConfig.domain_allowlist`.
diff --git a/src/soup_cli/commands/serve.py b/src/soup_cli/commands/serve.py
index f7303ea9..53575eb7 100644
--- a/src/soup_cli/commands/serve.py
+++ b/src/soup_cli/commands/serve.py
@@ -2316,10 +2316,22 @@ def _create_app(
raise HTTPException(status_code=400, detail="Invalid request")
if len(code) > tool_max_code_len:
raise HTTPException(status_code=400, detail="Invalid request")
- try:
- from soup_cli.trainer.rewards import _run_code_sandbox
+ from soup_cli.trainer.rewards import (
+ SandboxIsolationError,
+ _get_isolation_strategy,
+ _run_code_sandbox,
+ )
- stdout = _run_code_sandbox(code)
+ # Same requirement as /v1/tools/bash: run only where the host
+ # provides OS-level isolation (namespaces / sandbox-exec), and only
+ # when that isolation is actually set up for the run (strict).
+ isolation_detail = "python sandbox requires OS-level isolation"
+ if _get_isolation_strategy() == "best-effort":
+ raise HTTPException(status_code=501, detail=isolation_detail)
+ try:
+ stdout = _run_code_sandbox(code, strict_namespaces=True)
+ except SandboxIsolationError:
+ raise HTTPException(status_code=501, detail=isolation_detail)
except Exception as exc: # noqa: BLE001
logger.debug("/v1/tools/python sandbox error: %s", exc)
raise HTTPException(status_code=500, detail="Internal server error")
diff --git a/src/soup_cli/trainer/rewards.py b/src/soup_cli/trainer/rewards.py
index 3683328a..49954fd5 100644
--- a/src/soup_cli/trainer/rewards.py
+++ b/src/soup_cli/trainer/rewards.py
@@ -24,7 +24,7 @@ import tempfile
import threading
from collections.abc import Callable, Mapping
from dataclasses import dataclass
-from functools import wraps
+from functools import partial, wraps
from pathlib import Path
from typing import Any
@@ -375,6 +375,16 @@ class SandboxProcessResult:
launch_failed: bool = False
+class SandboxIsolationError(PermissionError):
+ """A strict sandbox run could not start its isolated child.
+
+ Raised by ``_run_code_sandbox(..., strict_namespaces=True)`` when the child
+ fails to launch because the Linux namespaces could not be set up in the
+ pre-exec hook. The code did not run. (A ``sandbox-exec`` wrapper that fails
+ on macOS is a non-zero exit of a launched child, not this error.)
+ """
+
+
def _run_sandboxed_subprocess(
argv: list[str],
preexec_fn: Callable[[], None] | None = None,
@@ -479,7 +489,7 @@ def _run_sandboxed_subprocess(
return SandboxProcessResult(proc.returncode, stdout, stderr)
-def _run_code_sandbox(code: str) -> "str | None":
+def _run_code_sandbox(code: str, strict_namespaces: bool = False) -> "str | None":
"""Run code in a subprocess sandbox with timeout, rlimits, and output caps.
Security posture (best-effort, NOT a strong sandbox):
@@ -491,17 +501,32 @@ def _run_code_sandbox(code: str) -> "str | None":
the child's default relative writes land in an ephemeral sandbox dir.
- Uses ``python -I -S`` to disable site packages and user customization.
+ ``strict_namespaces`` (default off) is for ``soup serve``'s
+ ``/v1/tools/python``: as in ``_run_bash_sandbox``, a failed namespace
+ ``unshare`` under the ``namespaces`` strategy stops the launch instead of
+ falling back, and a launch failure raises :class:`SandboxIsolationError`
+ instead of returning None. The reward and recipe callers keep the default,
+ which runs the code without the namespaces where they cannot be created.
+
Returns stdout string or None on failure.
"""
_show_code_exec_warning_once()
wrapped = SANDBOX_NETWORK_GUARD + "\n" + code
- preexec = _apply_rlimit if sys.platform != "win32" else None
+ preexec: Callable[[], None] | None = None
+ if sys.platform != "win32":
+ preexec = (
+ partial(_apply_rlimit, strict_namespaces=True)
+ if strict_namespaces
+ else _apply_rlimit
+ )
argv: list[str] = [sys.executable, "-I", "-S", "-c", wrapped]
result = _run_sandboxed_subprocess(argv, preexec)
+ if strict_namespaces and result.launch_failed:
+ raise SandboxIsolationError(result.stderr)
if result.returncode != 0 or result.timed_out or result.output_exceeded or result.launch_failed:
return None
return result.stdout
diff --git a/tests/test_serve_python_tool_isolation.py b/tests/test_serve_python_tool_isolation.py
new file mode 100644
index 00000000..9bcd5768
--- /dev/null
+++ b/tests/test_serve_python_tool_isolation.py
@@ -0,0 +1,292 @@
+"""soup serve: /v1/tools/python needs OS-level isolation, as /v1/tools/bash does.
+
+Where the code sandbox has only its in-process guards to offer (the
+``best-effort`` isolation strategy: Windows, or a Linux interpreter without
+``os.unshare``), the python route answers 501 and does not start the
+sandbox, the same answer the bash route gives there.
+
+Where the strategy is ``namespaces`` but the kernel or container refuses to
+create the namespaces at run time, the route gives the same 501 and the code
+does not start, as the bash route does. The code sandbox used by training
+(``code_exec_reward``) and by the recipe ``code`` node keeps running the code
+without the namespaces there, as before.
+"""
+
+from __future__ import annotations
+
+import errno
+import io
+import os
+import subprocess
+import sys
+import types
+from dataclasses import dataclass, field
+from unittest.mock import MagicMock
+
+import pytest
+
+pytest.importorskip("fastapi")
+pytest.importorskip("httpx")
+from fastapi.testclient import TestClient # noqa: E402
+
+PYTHON_DETAIL = "python sandbox requires OS-level isolation"
+BASH_DETAIL = "bash sandbox requires OS-level isolation"
+TOKEN = "s3cret-token"
+
+
+def _client(**kwargs) -> TestClient:
+ from soup_cli.commands.serve import _create_app
+
+ base = dict(
+ model_obj=MagicMock(),
+ tokenizer=MagicMock(),
+ device="cpu",
+ model_name="test-model",
+ max_tokens_default=128,
+ )
+ base.update(kwargs)
+ return TestClient(_create_app(**base), base_url="http://127.0.0.1")
+
+
+def sandbox_calls(monkeypatch) -> list[str]:
+ """Replace the code sandbox with a recorder that returns fixed stdout."""
+ calls: list[str] = []
+
+ def _fake_sandbox(code: str, strict_namespaces: bool = False) -> str:
+ calls.append(code)
+ return "ran"
+
+ monkeypatch.setattr("soup_cli.trainer.rewards._run_code_sandbox", _fake_sandbox)
+ return calls
+
+
+def _set_strategy(monkeypatch, strategy: str) -> None:
+ monkeypatch.setattr("soup_cli.trainer.rewards._get_isolation_strategy", lambda: strategy)
+
+
+def test_python_tool_refused_without_os_isolation(monkeypatch, sandbox_calls):
+ _set_strategy(monkeypatch, "best-effort")
+ resp = _client().post("/v1/tools/python", json={"code": "print(1)"})
+ assert resp.status_code == 501, resp.text
+ assert resp.json()["detail"] == PYTHON_DETAIL
+ assert sandbox_calls == []
+
+
[email protected]("strategy", ["namespaces", "sandbox-exec"])
+def test_python_tool_runs_with_os_isolation(monkeypatch, sandbox_calls, strategy):
+ _set_strategy(monkeypatch, strategy)
+ resp = _client().post("/v1/tools/python", json={"code": "print(1)"})
+ assert resp.status_code == 200, resp.text
+ assert resp.json() == {"stdout": "ran", "stderr": "", "exit_code": 0, "timed_out": False}
+ assert sandbox_calls == ["print(1)"]
+
+
+ ("path", "body", "detail"),
+ [
+ ("/v1/tools/python", {"code": "print(1)"}, PYTHON_DETAIL),
+ ("/v1/tools/bash", {"command": "echo 1"}, BASH_DETAIL),
+ ],
+)
+def test_code_tools_give_the_same_answer_without_os_isolation(
+ monkeypatch, sandbox_calls, path, body, detail
+):
+ _set_strategy(monkeypatch, "best-effort")
+ resp = _client().post(path, json=body)
+ assert resp.status_code == 501, (path, resp.text)
+ assert resp.json()["detail"] == detail
+
+
+def test_invalid_request_is_still_400_without_os_isolation(monkeypatch, sandbox_calls):
+ _set_strategy(monkeypatch, "best-effort")
+ resp = _client().post("/v1/tools/python", json={})
+ assert resp.status_code == 400, resp.text
+ assert resp.json()["detail"] == "Invalid request"
+
+
+def test_token_is_checked_before_isolation(monkeypatch, sandbox_calls):
+ _set_strategy(monkeypatch, "best-effort")
+ client = _client(auth_token=TOKEN)
+ resp = client.post(
+ "/v1/tools/python",
+ json={"code": "print(1)"},
+ headers={"Authorization": "Bearer wrong"},
+ )
+ assert resp.status_code == 401, resp.text
+ assert resp.json()["detail"] == "Invalid or missing bearer token"
+ assert sandbox_calls == []
+
+
+def test_correct_token_does_not_lift_the_isolation_requirement(monkeypatch, sandbox_calls):
+ _set_strategy(monkeypatch, "best-effort")
+ client = _client(auth_token=TOKEN)
+ resp = client.post(
+ "/v1/tools/python",
+ json={"code": "print(1)"},
+ headers={"Authorization": f"Bearer {TOKEN}"},
+ )
+ assert resp.status_code == 501, resp.text
+ assert resp.json()["detail"] == PYTHON_DETAIL
+ assert sandbox_calls == []
+
+
+# ---------------------------------------------------------------------------
+# The ``namespaces`` strategy, with the namespaces refused at run time.
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class _Child:
+ """What the stand-in for the forked sandbox child saw."""
+
+ unshare_error: Exception | None
+ unshare_flags: list[int] = field(default_factory=list)
+ rlimits: list[tuple] = field(default_factory=list)
+ started: list[str] = field(default_factory=list)
+
+
+def child(monkeypatch) -> _Child:
+ """Run the sandbox's pre-exec hook in this process instead of a forked child.
+
+ The sandbox sets resource limits and unshares the Linux namespaces in a
+ ``preexec_fn``, which runs in the forked child before the code starts. When
+ the hook raises, the code never starts and CPython's ``Popen`` raises
+ ``SubprocessError("Exception occurred in preexec_fn.")``. The stand-in
+ ``Popen`` below does the same here, so every process-wide call the hook
+ makes is replaced first: ``resource`` by a recorder, and ``os.unshare`` by
+ a recorder that raises ``child.unshare_error`` (EPERM by default, the answer
+ of a kernel or container that refuses unprivileged user namespaces). The
+ sandbox module sees a Linux host with the ``namespaces`` strategy on every
+ platform the suite runs on.
+ """
+ from soup_cli.trainer import rewards
+
+ state = _Child(unshare_error=PermissionError(errno.EPERM, "Operation not permitted"))
+
+ def _setrlimit(which, limits) -> None:
+ state.rlimits.append((which, limits))
+
+ fake_resource = types.SimpleNamespace(
+ setrlimit=_setrlimit,
+ RLIMIT_AS=9,
+ RLIMIT_CPU=0,
+ RLIMIT_FSIZE=1,
+ RLIMIT_CORE=4,
+ RLIMIT_NOFILE=7,
+ RLIMIT_NPROC=6,
+ )
+
+ def _unshare(flags: int) -> None:
+ state.unshare_flags.append(flags)
+ if state.unshare_error is not None:
+ raise state.unshare_error
+
+ class _Popen:
+ def __init__(self, argv, *, preexec_fn=None, **_kwargs) -> None:
+ if preexec_fn is not None:
+ # The hook must never reach the real limits or namespaces of
+ # the test process.
+ assert sys.modules["resource"] is fake_resource
+ assert os.unshare is _unshare
+ try:
+ preexec_fn()
+ except Exception as exc:
+ raise subprocess.SubprocessError(
+ "Exception occurred in preexec_fn."
+ ) from exc
+ state.started.append(argv[-1])
+ self.stdout = io.BytesIO(b"ran\n")
+ self.stderr = io.BytesIO(b"")
+ self.returncode = 0
+
+ def wait(self, timeout=None) -> int:
+ return self.returncode
+
+ def kill(self) -> None:
+ pass
+
+ monkeypatch.setitem(sys.modules, "resource", fake_resource)
+ monkeypatch.setattr(os, "unshare", _unshare, raising=False)
+ monkeypatch.setattr(
+ rewards, "sys", types.SimpleNamespace(platform="linux", executable=sys.executable)
+ )
+ monkeypatch.setattr(
+ rewards,
+ "subprocess",
+ types.SimpleNamespace(
+ Popen=_Popen,
+ PIPE=subprocess.PIPE,
+ SubprocessError=subprocess.SubprocessError,
+ TimeoutExpired=subprocess.TimeoutExpired,
+ ),
+ )
+ _set_strategy(monkeypatch, "namespaces")
+ return state
+
+
+def _namespace_flags() -> int:
+ from soup_cli.trainer import rewards
+
+ return rewards._CLONE_NEWUSER | rewards._CLONE_NEWNET | rewards._CLONE_NEWPID
+
+
+def test_python_tool_refused_when_the_namespaces_cannot_be_created(child):
+ resp = _client().post("/v1/tools/python", json={"code": "print(1)"})
+ assert resp.status_code == 501, resp.text
+ assert resp.json()["detail"] == PYTHON_DETAIL
+ assert child.unshare_flags == [_namespace_flags()]
+ assert child.started == []
+
+
+def test_python_and_bash_tools_agree_when_the_namespaces_cannot_be_created(child):
+ python_resp = _client().post("/v1/tools/python", json={"code": "print(1)"})
+ bash_resp = _client().post("/v1/tools/bash", json={"command": "echo 1"})
+ assert python_resp.status_code == 501, python_resp.text
+ assert bash_resp.status_code == 501, bash_resp.text
+ assert child.unshare_flags == [_namespace_flags()] * 2
+ assert child.started == []
+
+
+def test_python_tool_runs_when_the_namespaces_are_created(child):
+ child.unshare_error = None
+ resp = _client().post("/v1/tools/python", json={"code": "print(1)"})
+ assert resp.status_code == 200, resp.text
+ assert resp.json() == {"stdout": "ran", "stderr": "", "exit_code": 0, "timed_out": False}
+ assert child.unshare_flags == [_namespace_flags()]
+ assert len(child.started) == 1
+ assert child.started[0].endswith("\nprint(1)")
+ assert child.rlimits, "the resource limits are still set before the code starts"
+
+
+def test_strict_code_sandbox_raises_when_the_namespaces_cannot_be_created(child):
+ from soup_cli.trainer import rewards
+
+ with pytest.raises(rewards.SandboxIsolationError):
+ rewards._run_code_sandbox("print(1)", strict_namespaces=True)
+ assert child.unshare_flags == [_namespace_flags()]
+ assert child.started == []
+
+
+def test_reward_sandbox_still_runs_when_the_namespaces_cannot_be_created(child):
+ """Training's code_exec_reward keeps its fallback: no namespaces, the code still runs."""
+ from soup_cli.trainer import rewards
+
+ assert rewards._run_code_sandbox("print(1)") == "ran"
+ scores = rewards.code_exec_reward(
+ [[{"role": "assistant", "content": "print(1)"}]], expected=["ran"]
+ )
+ assert scores == [1.0]
+ assert child.unshare_flags == [_namespace_flags()] * 2
+ assert len(child.started) == 2
+
+
+def test_recipe_code_node_still_runs_when_the_namespaces_cannot_be_created(child):
+ from soup_cli.utils.recipe_dag import RecipeNode
+ from soup_cli.utils.recipe_run import _node_code
+
+ node = RecipeNode(name="c", kind="code", config={"code": "print(1)"})
+ assert _node_code(node, (({"x": 1},),)) == [{"x": 1, "c": "ran"}]
+ assert child.unshare_flags == [_namespace_flags()]
+ assert len(child.started) == 1
diff --git a/tests/test_v0536.py b/tests/test_v0536.py
index 75f902c9..35367cb9 100644
--- a/tests/test_v0536.py
+++ b/tests/test_v0536.py
@@ -486,16 +486,27 @@ def test_ngram_kwarg_skipped_when_draft_model_set():
"path",
["/v1/tools/python", "/v1/tools/web_search"],
)
-def test_tool_endpoint_returns_501(path: str):
+def test_tool_endpoint_returns_501(path: str, monkeypatch):
"""v0.53.7: python + web_search are live; the 501 stub is gone.
Per-endpoint live coverage lives in ``test_v0537.py``. The ``bash``
endpoint was reverted to 501 by the v0.53.7 review (C1) — its 501
surface is asserted in ``test_v0537.py::TestReviewFixesC1BashStub``.
+ The python route answers 501 where the host has no OS-level isolation
+ or the namespace ``unshare`` fails at run time
+ (``tests/test_serve_python_tool_isolation.py``), so this test runs it
+ under an isolating strategy whose ``unshare`` is a no-op that succeeds.
"""
+ import os
+
pytest.importorskip("fastapi")
from fastapi.testclient import TestClient
+ from soup_cli.trainer import rewards
+
+ if rewards._get_isolation_strategy() == "best-effort":
+ monkeypatch.setattr(rewards, "_get_isolation_strategy", lambda: "namespaces")
+ monkeypatch.setattr(os, "unshare", lambda *args, **kwargs: None, raising=False)
app = _build_app()
client = TestClient(app, base_url="http://127.0.0.1")
response = client.post(path, json={"code": "print(1)"})
diff --git a/tests/test_v0537.py b/tests/test_v0537.py
index 2e3be2eb..eaf3e6d7 100644
--- a/tests/test_v0537.py
+++ b/tests/test_v0537.py
@@ -897,14 +897,33 @@ def _create_test_app(host: str = "127.0.0.1", auth_token: str | None = None, **k
)
+def _allow_live_python_tool(monkeypatch) -> None:
+ """Let a live /v1/tools/python run proceed whatever this host's isolation.
+
+ The route answers 501 under the ``best-effort`` isolation strategy (Windows,
+ Linux without ``os.unshare``) and when the namespace ``unshare`` fails at run
+ time (a kernel or container that refuses unprivileged user namespaces).
+ These tests exercise the live sandbox run, not the isolation, so the
+ ``unshare`` the sandbox child calls is a no-op that succeeds.
+ """
+ import os
+
+ from soup_cli.trainer import rewards
+
+ if rewards._get_isolation_strategy() == "best-effort":
+ monkeypatch.setattr(rewards, "_get_isolation_strategy", lambda: "namespaces")
+ monkeypatch.setattr(os, "unshare", lambda *args, **kwargs: None, raising=False)
+
+
class TestToolEndpointsLive:
@pytest.fixture(autouse=True)
def _require_fastapi(self):
pytest.importorskip("fastapi")
pytest.importorskip("httpx")
- def test_python_tool_runs_simple_code(self):
+ def test_python_tool_runs_simple_code(self, monkeypatch):
from fastapi.testclient import TestClient
+ _allow_live_python_tool(monkeypatch)
app = _create_test_app()
client = TestClient(app, base_url="http://127.0.0.1")
resp = client.post(
@@ -1318,9 +1337,10 @@ class TestToolEndpointsLive:
assert any("example.com" in u for u in urls)
assert all("other.com" not in u for u in urls)
- def test_tools_no_longer_return_501(self):
+ def test_tools_no_longer_return_501(self, monkeypatch):
"""v0.53.7 #103 regression: 501-stubs are gone."""
from fastapi.testclient import TestClient
+ _allow_live_python_tool(monkeypatch)
app = _create_test_app()
client = TestClient(app, base_url="http://127.0.0.1")
# Python: should return 200 with sandbox response.
@@ -1879,13 +1899,15 @@ class TestReviewFixesSSEHeaders:
class TestReviewFixesAuthToken:
"""v0.53.7 H-A: optional Bearer-token gate on tool endpoints."""
- def test_python_tool_with_auth_token_requires_bearer(self):
+ def test_python_tool_with_auth_token_requires_bearer(self, monkeypatch):
try:
import fastapi # noqa: F401
except ImportError:
pytest.skip("FastAPI not installed")
from fastapi.testclient import TestClient
+ _allow_live_python_tool(monkeypatch)
+
from soup_cli.commands.serve import _create_app
app = _create_app(
@@ -1915,13 +1937,15 @@ class TestReviewFixesAuthToken:
)
assert resp.status_code == 200
- def test_auth_token_none_means_no_gate(self):
+ def test_auth_token_none_means_no_gate(self, monkeypatch):
try:
import fastapi # noqa: F401
except ImportError:
pytest.skip("FastAPI not installed")
from fastapi.testclient import TestClient
+ _allow_live_python_tool(monkeypatch)
+
from soup_cli.commands.serve import _create_app
app = _create_app(