ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.
Python ai-agentsamdcomfyuidockerllama-cppllmlocal-ain8nnvidiaopen-webuiragself-hostedspeech-to-textstrix-halotext-to-speechworkflow-automation
import json
import os
from pathlib import Path
import subprocess
import sys
import tempfile
import threading
import time
import unittest
from contextlib import ExitStack
from unittest.mock import patch
HOST = Path(__file__).resolve().parents[1] / "host"
sys.path.insert(0, str(HOST))
from project_controller import ProjectController
from project_artifacts import MissingProjectOutput, RejectedProjectArtifacts, InvalidProjectArtifacts
@unittest.skipUnless(os.name == "posix", "POSIX candidate coordinator")
class ProjectControllerPolicyTests(unittest.TestCase):
def test_missing_output_finishes_failed_and_cleans_up_without_import(self):
self.check_collection_failure(MissingProjectOutput('missing dist'), 'failed')
def test_rejected_archive_finishes_failed_but_transport_uncertainty_stays_unconfirmed(self):
self.check_collection_failure(RejectedProjectArtifacts('invalid artifact path'), 'failed')
self.check_collection_failure(InvalidProjectArtifacts('artifact stream interrupted'), 'unconfirmed')
def check_collection_failure(self, error, expected):
with tempfile.TemporaryDirectory() as root:
controller = ProjectController(root, Path(root) / 'state', 'sha256:' + 'a' * 64,
authorize=lambda *args: True)
request = {'project': 'project', 'sourceSha256': 'b' * 64,
'image': controller.image, 'outputDirectory': 'dist'}
job, _ = controller.jobs.create('a' * 64, request)
try:
with ExitStack() as mocks:
mocks.enter_context(patch.object(controller.storage, 'reserve'))
mocks.enter_context(patch.object(controller.storage, 'create_volume'))
mocks.enter_context(patch('project_controller.start_keeper'))
mocks.enter_context(patch('project_controller.subprocess.run', side_effect=[
subprocess.CompletedProcess([], 1), subprocess.CompletedProcess([], 0)]))
mocks.enter_context(patch('project_controller.seed_project'))
mocks.enter_context(patch('project_controller.run_stage', return_value={'status':'succeeded','exitCode':0}))
mocks.enter_context(patch('project_controller.collect_artifacts', side_effect=error))
imported = mocks.enter_context(patch('project_controller.import_artifacts'))
cleaned = mocks.enter_context(patch.object(controller, '_cleanup', return_value=[]))
controller._work(job, request, {}, threading.Event())
imported.assert_not_called()
if expected == 'failed':
cleaned.assert_called_once()
else:
cleaned.assert_not_called()
self.assertEqual(controller.jobs.observe(job)['state'], expected)
finally:
controller.close()
def test_denied_request_does_not_read_source_or_execute(self):
with tempfile.TemporaryDirectory() as root:
controller = ProjectController(root, Path(root) / "state", "sha256:" + "a" * 64,
authorize=lambda *args: False)
try:
with patch("project_controller.snapshot_project") as snapshot, patch("project_controller.subprocess.run") as command:
with self.assertRaises(PermissionError):
controller.submit("a" * 64, "project")
snapshot.assert_not_called()
command.assert_not_called()
finally:
controller.close()
def test_no_default_authorization_adapter(self):
with tempfile.TemporaryDirectory() as root:
with self.assertRaises(ValueError):
ProjectController(root, Path(root) / "state", "sha256:" + "a" * 64, authorize=None)
def test_controller_exception_is_unconfirmed_and_not_reclaimed(self):
with tempfile.TemporaryDirectory() as root:
controller = ProjectController(root, Path(root) / "state", "sha256:" + "a" * 64,
authorize=lambda *args: True)
request = {"project": "project", "sourceSha256": "b" * 64,
"image": controller.image, "outputDirectory": "out"}
job, _ = controller.jobs.create("a" * 64, request)
try:
with patch("project_controller.subprocess.run", side_effect=subprocess.TimeoutExpired("docker", 15)):
controller._work(job, request, {}, threading.Event())
self.assertEqual(controller.observe(job)["state"], "unconfirmed")
self.assertFalse(controller.jobs.claim(job))
finally:
controller.close()
def test_revocation_or_cancellation_during_collection_prevents_import(self):
for change in ("revoke", "cancel"):
with self.subTest(change=change), tempfile.TemporaryDirectory() as root:
allowed = [True]
cancel = threading.Event()
controller = ProjectController(root, Path(root) / "state", "sha256:" + "a" * 64,
authorize=lambda *args: allowed[0])
request = {"project": "project", "sourceSha256": "b" * 64,
"image": controller.image, "outputDirectory": "out"}
job, _ = controller.jobs.create("a" * 64, request)
def collect(*args):
if change == "revoke":
allowed[0] = False
else:
cancel.set()
return {}
try:
with ExitStack() as mocks:
mocks.enter_context(patch.object(controller.storage, 'reserve'))
mocks.enter_context(patch.object(controller.storage, 'create_volume'))
mocks.enter_context(patch('project_controller.start_keeper'))
mocks.enter_context(patch("project_controller.subprocess.run", side_effect=[
subprocess.CompletedProcess([], 1), subprocess.CompletedProcess([], 0)]))
mocks.enter_context(patch("project_controller.seed_project"))
mocks.enter_context(patch("project_controller.run_stage", return_value={"status": "succeeded", "exitCode": 0}))
mocks.enter_context(patch("project_controller.collect_artifacts", side_effect=collect))
imported = mocks.enter_context(patch("project_controller.import_artifacts"))
mocks.enter_context(patch.object(controller, "_cleanup", return_value=[]))
controller._work(job, request, {}, cancel)
imported.assert_not_called()
result = controller.jobs.observe(job)
self.assertEqual(result["state"], "unconfirmed" if change == "revoke" else "cancelled")
self.assertFalse(controller.jobs.claim(job))
finally:
controller.close()
def test_uncertain_docker_creation_retains_bounded_volume_and_reservation(self):
for uncertain in ('keeper', 'seed', 'stage'):
with self.subTest(uncertain=uncertain), tempfile.TemporaryDirectory() as root:
controller = ProjectController(root, Path(root) / 'state', 'sha256:' + 'a' * 64,
authorize=lambda *_: True)
request = {'project': 'project', 'sourceSha256': 'b' * 64,
'image': controller.image, 'outputDirectory': 'out'}
job, _ = controller.jobs.create('a' * 64, request)
try:
with ExitStack() as mocks:
mocks.enter_context(patch.object(controller.storage, 'reserve'))
mocks.enter_context(patch.object(controller.storage, 'create_volume'))
mocks.enter_context(patch('project_controller.start_keeper',
side_effect=subprocess.TimeoutExpired('docker', 15) if uncertain == 'keeper' else None))
mocks.enter_context(patch('project_controller.seed_project',
side_effect=subprocess.TimeoutExpired('docker', 70) if uncertain == 'seed' else None))
mocks.enter_context(patch('project_controller.run_stage',
return_value={'status': 'unconfirmed', 'exitCode': None}))
cleanup = mocks.enter_context(patch.object(controller, '_cleanup', return_value=[]))
controller._work(job, request, {}, threading.Event())
self.assertEqual(controller.jobs.observe(job)['state'], 'unconfirmed')
cleanup.assert_not_called()
finally:
controller.close()
def test_terminal_cancel_retries_owned_cleanup_without_rewriting_outcome(self):
with tempfile.TemporaryDirectory() as root:
controller = ProjectController(root, Path(root) / 'state', 'sha256:' + 'a' * 64,
authorize=lambda *_: True)
request = {'project': 'project', 'sourceSha256': 'b' * 64,
'image': controller.image, 'outputDirectory': 'out'}
job, _ = controller.jobs.create('a' * 64, request)
controller.jobs.claim(job)
controller.jobs.controller_failure(job, 'test failed', state='failed')
controller.jobs.cleanup_warnings(job, ['storage capacity remains reserved'])
controller.image = 'sha256:' + 'c' * 64
try:
with patch.object(controller, '_cleanup', return_value=[]) as cleanup:
controller.cancel(job)
controller.close()
cleanup.assert_called_once_with(job, image=request['image'], runtime='npm')
row = controller.jobs.observe(job)
self.assertEqual(row['state'], 'failed')
self.assertEqual(row['output']['error'], 'test failed')
finally:
controller.close()
@unittest.skipUnless(os.environ.get("ODS_TEST_PROJECT_NODE") == "1", "real Docker opt-in")
class ProjectControllerRuntimeTests(unittest.TestCase):
def test_complete_pipeline_and_replay_after_reopen(self):
with tempfile.TemporaryDirectory() as root:
root = Path(root)
iid = root / "image"
subprocess.run(["docker", "build", "--iidfile", str(iid), "-f",
str(HOST / "Dockerfile.project-node"), str(HOST)],
check=True, capture_output=True, timeout=180)
image = iid.read_text().strip()
project = root / "project"
project.mkdir()
package = {"private": True, "scripts": {"test": "node check.cjs", "build": "node build.cjs"}}
(project / "package.json").write_text(json.dumps(package))
(project / "package-lock.json").write_text(json.dumps({"lockfileVersion": 3, "packages": {"": package}}))
(project / "check.cjs").write_text("require('assert').notEqual(process.getuid(),0)")
(project / "build.cjs").write_text("const f=require('fs');f.mkdirSync('out');f.writeFileSync('out/index.html','<h1>Controller built</h1>')")
# Test-only explicit scope; no production policy or receipt is changed.
def authorized(selected, action, binding):
return selected == "project" and action in {"snapshot", "execute", "import", "observe"}
controller = ProjectController(root, root / "state", image, authorize=authorized)
original_run = subprocess.run
records = []
diagnostic_command = ['docker', 'info', '--format', '{{.MemTotal}}']
def byte_length(value):
if value is None:
return 0
if isinstance(value, (bytes, bytearray)):
return len(value)
if isinstance(value, str):
return len(value.encode('utf-8', 'replace'))
try:
return len(value)
except TypeError:
return 0
def observer(args, *pargs, **kwargs):
if not (isinstance(args, (list, tuple)) and len(args) == len(diagnostic_command)
and all(a == b for a, b in zip(args, diagnostic_command))):
return original_run(args, *pargs, **kwargs)
started = time.monotonic()
record = {'cause': 'observed', 'elapsedSeconds': None, 'returnCode': None,
'errno': None, 'timeout': None, 'stdoutBytes': 0, 'stderrBytes': 0}
try:
result = original_run(args, *pargs, **kwargs)
except subprocess.TimeoutExpired as exc:
record['cause'] = 'timeout'
timeout = getattr(exc, 'timeout', None)
if timeout is None:
timeout = kwargs.get('timeout')
record['timeout'] = timeout if isinstance(timeout, (int, float)) else None
record['stdoutBytes'] = byte_length(getattr(exc, 'stdout', None))
record['stderrBytes'] = byte_length(getattr(exc, 'stderr', None))
raise
except subprocess.CalledProcessError as exc:
record['cause'] = 'called-process-error'
record['returnCode'] = exc.returncode if isinstance(exc.returncode, int) else None
record['stdoutBytes'] = byte_length(getattr(exc, 'stdout', None))
record['stderrBytes'] = byte_length(getattr(exc, 'stderr', None))
raise
except OSError as exc:
record['cause'] = 'oserror'
errno = getattr(exc, 'errno', None)
record['errno'] = errno if isinstance(errno, int) else None
raise
finally:
record['elapsedSeconds'] = round(time.monotonic() - started, 6)
records.append(record)
record['returnCode'] = result.returncode if isinstance(getattr(result, 'returncode', None), int) else None
record['stdoutBytes'] = byte_length(getattr(result, 'stdout', None))
record['stderrBytes'] = byte_length(getattr(result, 'stderr', None))
return result
try:
with patch("project_controller.subprocess.run", observer):
try:
accepted = controller.submit("a" * 64, "project")
finally:
controller.close()
except BaseException as exc:
if records:
exc.add_note("memtotal-observer: " + json.dumps(records, sort_keys=True))
raise
result = controller.observe(accepted["id"])
diagnostic = "memtotal-observer: " + json.dumps(records, sort_keys=True)
self.assertEqual(result["state"], "succeeded", {'result': result, 'engineInfo': records})
self.assertNotIn("cleanupWarnings", result["output"], diagnostic)
self.assertEqual([s["stage"] for s in result["steps"]], ["acquire", "test", "build"], diagnostic)
destination = root / result["output"]["relativeDirectory"] / "index.html"
self.assertEqual(destination.read_bytes(), b"<h1>Controller built</h1>", diagnostic)
self.assertFalse((project / "out").exists(), diagnostic)
reopened = ProjectController(root, root / "state", image, authorize=authorized)
try:
replay = reopened.submit("a" * 64, "project")
self.assertEqual(replay["id"], result["id"], diagnostic)
self.assertEqual(replay["state"], "succeeded", diagnostic)
self.assertFalse(reopened.futures, diagnostic)
finally:
reopened.close()