Skip to content

Self-improving AI Employees for everyone

docs(CLAUDE.md): refresh stale paths + add 5 cross-cutting invariants

trohitg635e4a3aa675
diff --git a/.env.template b/.env.template
index 72d810a2..fd9acc26 100644
--- a/.env.template
+++ b/.env.template
@@ -51,7 +51,58 @@ TEMPORAL_NAMESPACE=default
TEMPORAL_TASK_QUEUE=machina-tasks
TEMPORAL_PER_TYPE_DISPATCH=true
TEMPORAL_AGENT_WORKFLOW_ENABLED=true
+# Persistence backend for the supervised Temporal cluster.
+# postgres (default) -- pgserver + temporal-server binary. Data lands
+# under ``DATA_DIR/postgres/`` (cross-platform
+# binary bundle, no Docker required).
+# sqlite -- single ``temporal api`` process. In-memory
+# (lost on restart); requires the ``temporal``
+# CLI on PATH (brew / scoop / cargo / ``npm
+# install -g temporal-server``).
TEMPORAL_BACKEND=postgres
+# Bind Temporal services to localhost only (true) or all interfaces
+# (false). Keep ``true`` for dev / single-host installs.
+TEMPORAL_BIND_LOCAL_ONLY=true
+# Pinned Temporal server binary version. Pooch downloads + SHA-256
+# verifies from temporalio/temporal GitHub releases on first boot.
+TEMPORAL_BINARY_VERSION=1.31.0
+# Graceful shutdown window for the Temporal child processes.
+TEMPORAL_GRACEFUL_SHUTDOWN_SECONDS=30
+# Temporal persistence layout (all relative to DATA_DIR unless
+# absolute). Override to point at an alternate disk / network mount.
+# - sqlite_path: SQLite db file ``temporal server start-dev`` opens
+# - postgres_subdir: pgserver data directory (postgres backend)
+# - server_subdir: cwd for the temporal-server process (config YAML)
+TEMPORAL_SQLITE_PATH=temporal/temporal.db
+TEMPORAL_POSTGRES_SUBDIR=postgres
+TEMPORAL_SERVER_SUBDIR=_temporal
+# Advanced tuning (uncomment to override defaults):
+# TEMPORAL_FRONTEND_GRPC_PORT=7233
+# TEMPORAL_MATCHING_GRPC_PORT=7235
+# TEMPORAL_HISTORY_GRPC_PORT=7234
+# TEMPORAL_WORKER_GRPC_PORT=7239
+# TEMPORAL_NUM_HISTORY_SHARDS=4
+# TEMPORAL_DEFAULT_MAX_CONNS=20
+# TEMPORAL_VISIBILITY_MAX_CONNS=4
+# TEMPORAL_MAX_CONN_LIFETIME=5m
+# TEMPORAL_POSTGRES_DSN= # override pgserver-managed DSN
+# TEMPORAL_PG_READY_TIMEOUT_SECONDS=60
+# TEMPORAL_SERVER_READY_TIMEOUT_SECONDS=120
+
+# Wave 12 event framework. ``true`` (default) routes trigger events
+# through TriggerListenerWorkflow + Temporal Signal fan-out for the
+# 8 canary-registered trigger types (webhook, chat, task, telegram,
+# whatsapp, gmail, cron, email). Flip to ``false`` to roll back to
+# the legacy event_waiter path -- see docs-internal/event_framework.md.
+EVENT_FRAMEWORK_ENABLED=true
+
+# Memory compaction. ``true`` (default) auto-compacts agent
+# conversation memory when cumulative tokens exceed the model's
+# context-window threshold. Per-session overrides via the UI.
+COMPACTION_ENABLED=true
+
+# Dead-letter queue for failed workflow executions. Off by default.
+DLQ_ENABLED=false
# Data directory (base for all persistent storage: DBs, workspaces,
# workflows, claude state). User-home rooted by default — same
diff --git a/CLAUDE.md b/CLAUDE.md
index fcd5e629..467ee4a3 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -169,6 +169,7 @@ server/core/
├── cache.py # CacheService with Redis/SQLite/Memory fallback
├── config.py # Application configuration
├── logging.py # Logging configuration
+├── paths.py # SSOT for on-disk locations: machina_root()/workspaces_dir()/whatsapp_dir() under ~/.machina/; packages_dir()/package_dir(name) via platformdirs (OS cache) for downloaded binaries; example_workflows_dir() = <repo>/.machina/workflows/ (shipped seeds, NOT under DATA_DIR)
├── encryption.py # Fernet encryption with PBKDF2 key derivation
├── credentials_database.py # Async SQLite for encrypted API keys and OAuth tokens
└── credential_backends.py # Multi-backend abstraction (Fernet, Keyring, AWS)
@@ -1399,9 +1400,9 @@ Interactive browser automation via the [agent-browser](https://www.npmjs.com/pac
| `server/services/handlers/browser.py` | Handler dispatcher mapping operation+params to agent-browser CLI args |
| `server/skills/web_agent/browser-skill/SKILL.md` | AI agent skill documenting snapshot-act-snapshot loop |
-**Installation Requirement:** `agent-browser` is a pinned project dependency in [package.json](../package.json). `pnpm install` places it under `node_modules/.pnpm/`, and `scripts/install.js` runs `npx agent-browser install` during postinstall to download the Chromium runtime. The handler returns an installation-instruction error if the binary cannot be located.
+**Installation Requirement:** `agent-browser` is MachinaOs-managed (NOT a `package.json` dep). On first browser-node invocation, [`server/nodes/browser/_install.py::agent_browser_binary_path`](../server/nodes/browser/_install.py) runs `npm install agent-browser@latest --prefix <package_dir("browser")/npm>` then `<bin> install` for the Chromium runtime. Same shape as [`claude_binary_path`](../server/nodes/agent/claude_code_agent/_oauth.py) (the precedent). Cache root: [`core.paths.packages_dir()`](../server/core/paths.py) → `~/.cache/MachinaOs/browser/` on Linux, `~/Library/Caches/MachinaOs/browser/` on macOS, `%LOCALAPPDATA%\MachinaOs\Cache\browser\` on Windows (XDG-conventional via `platformdirs.user_cache_path`).
-**Implementation Detail:** `browser_service.py` invokes the binary via `[shutil.which("npx"), "--no-install", "agent-browser", ...args]` -- the same pattern used by [claude_code_service.py](../server/services/claude_code_service.py). This avoids custom cross-platform shim detection: `npx --no-install` handles local `node_modules/.bin/` resolution, Windows/POSIX shim differences, and shebang interpretation internally. The service reads only the first JSON output line from agent-browser (the daemon holds stdout open), truncates at 100KB, and kills the process tree via `psutil`. All subprocess calls use `shell=False` with list argv to avoid BatBadBut (CVE-2024-1874) on Windows.
+**Implementation Detail:** `browser_service.py` invokes the resolved binary directly via `subprocess.Popen([binary, ...args], shell=False)`. Reads only the first JSON output line from agent-browser (the daemon holds stdout open), truncates at 100KB, and kills the process tree via `psutil`. `shell=False` with list argv avoids BatBadBut (CVE-2024-1874) on Windows; `npm` itself is located via `shutil.which("npm")` which honours `PATHEXT` for the `.cmd` shim.
---
@@ -3199,17 +3200,17 @@ aws = ["boto3>=1.34.0"] # AWS Secrets Manager
## Example Workflows
### Overview
-Example workflows are pre-built workflow templates that auto-load on first use. They provide users with starting points to explore the platform's capabilities. Examples are stored as JSON files in the `workflows/` folder at the project root.
+Example workflows are pre-built workflow templates that auto-load on first use. They provide users with starting points to explore the platform's capabilities. Seeds live as JSON files at **`<repo>/.machina/workflows/`** — the only git-tracked content under `<repo>/.machina/` (everything else there is `.gitignore`d as runtime state). The path is also preserved by `machina clean` (see `_MACHINA_KEEP` in `cli/commands/clean.py`).
### Architecture
```
-workflows/ # Example workflow JSON files (project root)
-├── hello_world.json
-├── zeenie_chat.json
-└── ...
+<repo>/.machina/workflows/ # Shipped seed JSONs (git-tracked)
+├── AI Assistant_example_workflow-*.json
+├── AI Employee_example_workflow-*.json
+└── Claude Assistant_example_workflow-*.json
server/services/
-└── example_loader.py # Loads and imports examples
+└── example_loader.py # Loads and imports examples via core.paths.example_workflows_dir()
server/models/database.py # UserSettings.examples_loaded flag
server/core/database.py # Migration for examples_loaded column
@@ -3218,7 +3219,7 @@ server/routers/database.py # Auto-load logic in get_all_workflows
### How It Works
1. **First Fetch Detection**: When `get_all_workflows` API is called, it checks `UserSettings.examples_loaded`
-2. **Auto-Import**: If `examples_loaded=false`, imports all JSON files from `workflows/` folder
+2. **Auto-Import**: If `examples_loaded=false`, calls `example_loader.get_example_workflows()` which reads from `core.paths.example_workflows_dir()` = `<repo>/.machina/workflows/`
3. **Mark Complete**: Sets `examples_loaded=true` to prevent re-import on subsequent fetches
4. **Anonymous Support**: Uses `user_id="default"` when `VITE_AUTH_ENABLED=false`
@@ -3259,7 +3260,8 @@ Example workflows use the same format as UI exports:
### Key Files
| File | Description |
|------|-------------|
-| `workflows/*.json` | Example workflow JSON files |
+| `.machina/workflows/*.json` | Shipped seed workflow JSONs (git-tracked) |
+| `server/core/paths.py` | `example_workflows_dir()` → `<repo>/.machina/workflows/` (fixed path, NOT under `DATA_DIR`) |
| `server/services/example_loader.py` | `get_example_workflows()`, `import_examples_for_user()` |
| `server/models/database.py` | `UserSettings.examples_loaded` field |
| `server/core/database.py` | Migration adds `examples_loaded` column |
@@ -3268,10 +3270,12 @@ Example workflows use the same format as UI exports:
### Example Loader Service
```python
# server/services/example_loader.py
-EXAMPLES_DIR = Path(__file__).parent.parent.parent / "workflows"
+from core.paths import example_workflows_dir
def get_example_workflows() -> List[Dict[str, Any]]:
"""Load all example workflow JSON files from disk."""
+ examples_dir = example_workflows_dir()
+ ...
async def import_examples_for_user(database) -> int:
"""Import all examples using existing database.save_workflow().
@@ -3295,9 +3299,9 @@ if not settings or not settings.get("examples_loaded", False):
### Adding Custom Examples
1. Export a workflow from the UI (File > Export)
-2. Copy the JSON file to `workflows/` folder at project root
+2. Copy the JSON file to `<repo>/.machina/workflows/` (git-tracked seed location)
3. Edit the `id` and `name` fields as needed
-4. Delete `server/machina.db` (or set `examples_loaded=false` in database)
+4. Delete `~/.machina/workflow.db` (or set `examples_loaded=false` in DB)
5. Restart server - examples auto-load on first workflow list fetch
### Database Migration
@@ -5201,7 +5205,7 @@ This function:
- **Node Data Architecture**: `node.data` only stores `label` (display name). All parameters are stored in the database via `save_node_parameters` WebSocket handler. This prevents parameter bloat in workflow JSON exports and keeps React Flow state lightweight. `useDragAndDrop.ts` saves default parameters to DB on drop, not to `node.data`.
- **Workflow Export/Import with Parameters**: Exported workflow JSON includes a `nodeParameters` field containing all node configuration (provider, model, prompt, skillsConfig, etc.) fetched from the database at export time. On import, embedded `nodeParameters` are saved back to the database. `sanitizeNodes()` in `workflowExport.ts` still strips `node.data` to UI-only fields (`label`, `disabled`, `condition`). A `parameterSanitizer.ts` utility exists for credential stripping but is currently disabled (pass-through). Old exports without `nodeParameters` import cleanly (backward compatible). Key files: `client/src/utils/workflowExport.ts`, `client/src/utils/parameterSanitizer.ts`, `client/src/Dashboard.tsx` (export/import handlers), `server/services/example_loader.py`.
- **Skill System Architecture**: Skills organized in `server/skills/<folder>/` subfolders. Each folder appears in Master Skill dropdown. DB is source of truth for skill instructions (seeded from SKILL.md on first load). Icon resolution mirrors `BaseNode._metadata_dict`: per-plugin `<plugin>/icon.svg` (served as `/api/schemas/nodes/<type>/icon`) → `visuals.json` (emoji / `lobehub:<brand>`); color: per-plugin `<plugin>/meta.json` → `visuals.json`. Lives in [`skill_loader.py::_parse_skill_metadata`](./server/services/skill_loader.py). Native DOM keydown handler prevents React Flow from intercepting Ctrl shortcuts in skill editor. **Mutation broadcasts use the CloudEvents-typed `skill_lifecycle` wire key** with stages `created` / `updated` / `deleted` / `content_saved` — see "Plugin-folder location for plugin-specific CloudEvents factories" below.
-- **Example Workflows**: Auto-load example workflows from `workflows/` folder on first use. Uses `UserSettings.examples_loaded` flag to track import status. Supports anonymous users (`user_id="default"`). Reuses existing `database.save_workflow()` for import. Embedded `nodeParameters` in example JSON files are saved to the database on import. See "Example Workflows" section for details.
+- **Example Workflows**: Auto-load example workflow seeds from `<repo>/.machina/workflows/` on first use (git-tracked; the only non-ignored content under `<repo>/.machina/`). Path resolved by `core.paths.example_workflows_dir()` — fixed, NOT under `DATA_DIR`, preserved by `machina clean` via `_MACHINA_KEEP`. Uses `UserSettings.examples_loaded` flag; supports anonymous users (`user_id="default"`); embedded `nodeParameters` saved to DB on import.
- **Onboarding Service**: 5-step welcome wizard (Welcome, Concepts, API Keys, Canvas Tour, Get Started) using Ant Design Steps/Card/Button/Typography. Database-backed via `UserSettings.onboarding_completed` + `onboarding_step`. Existing users auto-skip (migration marks `examples_loaded=1` as completed). Replayable from Settings "Help" section. No new WebSocket handlers needed. See [Onboarding Service](./docs-internal/onboarding.md) for details.
- **Node.js Code Executor**: Persistent Node.js server (Express + tsx) at port 3020 for JavaScript/TypeScript execution, replacing subprocess spawning per execution. Handlers in `server/services/handlers/code.py` call `NodeJSClient` which makes HTTP requests to the Node.js server. All config via environment variables (`NODEJS_EXECUTOR_URL`, `NODEJS_EXECUTOR_PORT`, etc.).
- **writeTodos Tool Node**: Dedicated AI tool for task planning connecting to any agent's `input-tools` handle. `TodoService` singleton (`server/services/todo_service.py`) stores JSON-based per-session todo state keyed by workflow_id. Handler broadcasts `phase: "todo_update"` via WebSocket on each update for real-time UI; `formatTodoOutput()` in `OutputDisplayPanel.tsx` renders the result as a checklist with `[ ]` / `[~]` / `[x]` icons. Schema uses `TodoItem`/`TodoStatus` Pydantic enum. Skill at `server/skills/assistant/write-todos-skill/SKILL.md` teaches the plan-work-update loop.
@@ -5233,5 +5237,10 @@ This function:
- **Telegram owner detection — three layers** (f746ddf): the bot owner is captured by THREE independent paths so the realistic setup flows all converge to a working state. (1) **Explicit field**: secondary `FieldDef` in the credentials modal lets the user paste their Telegram user ID directly (recommended; works without DM'ing the bot). (2) **Pre-poll peek** in `connect()` via `_capture_owner_from_pending_updates()` — calls `bot.get_updates(timeout=0)` BEFORE `start_polling(drop_pending_updates=True)` discards the queue, scans for any historical private DM, captures + persists atomically. (3) **Atomic write-through in `_on_message_received`** — persists FIRST, sets in-memory ONLY on success. Invariant: "in-memory has owner ⇒ DB has owner" so a process restart can re-capture cleanly. Failure logged at `ERROR` with `exc_info=True` (was `WARNING` previously, masking persist failures). The lazy fallback in `telegram_send.py` (read DB → `service.set_owner`) sits on top of all three.
- **Supervisor Job Object loud-failure (machina/tree.py + supervisor.py)** (8a64eb9): the Windows process-tree mechanism depends entirely on a Job Object with `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE` (no fallback — `start_new_session` is POSIX-only). `pywin32>=308` is a hard supervisor dependency on Windows; the declaration in `machina/pyproject.toml` carries an inline comment so future maintainers don't drop it. `_JobObject.__init__` splits failures into explicit `ImportError` (with reinstall command) and `Exception` (with `repr`) branches — both write to `stderr` instead of swallowing. `_JobObject.add()` verifies enrollment via `win32job.IsProcessInJob(handle, self._handle)` after assignment; on mismatch it queries `IsProcessInJob(handle, None)` so the warning explicitly names the wrapper-job hypothesis (npm / pnpm / conhost wrappers occasionally place us in their own non-nesting Job). `supervisor._spawn_once` checks `add_to_job` return value and emits a yellow `WARN: pid=N not enrolled in Job Object` per child if False. Without these guards, a stale pywin32 install or a wrapping Job Object silently leaks orphan Python processes on every Ctrl-C and they accumulate to the point of holding SQLite locks that block subsequent backend startup.
- **Credentials envelope-shape invariant** (dc94cde): the `useCredentialPanel` query stores `{values: CredentialFormValues, hadStored: boolean}` (an envelope, NOT a flat dict). The earlier `writeValues` called `setQueryData<CredentialFormValues>` with a spread `(prev) => ({...prev, [key]: value})`, which at runtime merged the typed character at the envelope level next to `values` and `hadStored` instead of inside `.values` — the input selector `panel.values[field.key]` re-rendered with the original (server) value on every keystroke and the input felt frozen. Fix: `writeValues` now preserves the envelope shape and updates only the inner `values` dict; `hadStored` is preserved verbatim (it reflects real server state, not local edits). When introducing other queries shaped as `{data, meta}` envelopes, follow the same pattern — never spread `prev` as if it were the inner payload.
+- **`core.paths` is the SSOT for on-disk locations**: state vs. install split — `machina_root()` (= `~/.machina/`) holds *state* (workspaces, whatsapp session DB, `credentials.db`, `workflow.db`, claude OAuth at `~/.machina/claude/`); `packages_dir()` (= `platformdirs.user_cache_path("MachinaOs")` → `~/.cache/MachinaOs/` on Linux, `~/Library/Caches/MachinaOs/` on macOS, `%LOCALAPPDATA%\MachinaOs\Cache\` on Windows) holds *downloaded binaries* via `package_dir(name)` per service (`stripe`, `claude`, `browser`). Wiping `packages_dir` forces a fresh redownload without dropping auth state. Out of scope: globally-installed binaries (`temporal`, Himalaya) and npm `package.json` deps (`edgymeow`). Shipped seed workflows are the lone exception — `example_workflows_dir()` is hardcoded to `<repo>/.machina/workflows/` (git-tracked seeds, NOT under `DATA_DIR`) so they survive `machina clean` and stay at the same path across `DATA_DIR=~/.machina` vs `DATA_DIR=.machina` configs.
+- **`machina clean` preserves `.machina/workflows/`**: `cli/commands/clean.py` iterates `<repo>/.machina/` children and skips anything in `_MACHINA_KEEP = frozenset({"workflows"})`. Wipes `claude/`, `workspaces/`, `*.db` as before; preserves the shipped seed JSONs so a fresh clone + clean doesn't lose user-importable examples. Test `cli/tests/test_clean.py::test_machina_keep_preserves_workflows` locks the keep-list.
+- **No raw `print()` outside three sanctioned helpers**: `main._startup_log` (pre-logger boot markers), `core.container._clog` (DI-bootstrap markers), and `nodes.code.python_executor.captured_print` (the sandbox builtin handed to user code). Everything else goes through `logger = get_logger(__name__)`. The supervisor prefixes every aggregated line with `[HH:MM:SS.fff]` so no inner `TimeStamper` is needed; structlog console mode is deliberately timestamp-less. Test `server/tests/test_no_raw_prints.py` AST-walks the tree and flags any unsanctioned `print(...)` call.
+- **`configure_logging(settings)` must run BEFORE plugin self-registration imports**: in `main.py`, `Settings()` + `configure_logging(settings)` + `init_tracing()` + `get_logger(__name__)` happen ahead of `from core.container import container` and `from routers import …`. Otherwise plugin folders that register on import call `logger.debug(...)` while structlog is still on its default processor chain (which includes `TimeStamper` + no `filter_by_level`) — symptom is double timestamps + debug records leaking despite `LOG_LEVEL=INFO`.
+- **Plugin-identifier shape validator (`services/plugin/identifiers.py`)** (`7700f87`): `NODE_TYPE_PATTERN = r"^[A-Za-z_][A-Za-z0-9_]*$"` + `is_valid_node_type(value)`. Single source of truth used by FastAPI URL routes (`Path(pattern=NODE_TYPE_PATTERN)` in `routers/schemas.py`) AND internal helpers (`nodes/_visuals.py::get_plugin_meta` / `get_plugin_icon_path`). Closes CodeQL `py/path-injection` alerts on the icon-resolution endpoints: the registry lookup `get_node_class(node_type)` already gates the taint in practice, but CodeQL can't follow registry-mediated sanitization — adding `fullmatch` at the function boundary is the canonical pattern its rule docs prescribe. Contract test (`tests/services/test_identifiers.py`, 47 cases) locks the regex against `../etc/passwd`, `..\windows\system32`, `foo\x00bar`, `%2e%2e%2fetc`, `foo;bar`, `${HOME}`, `` foo`bar ``, CRLF, and non-string input.
- **Wave 13 canary fixes** — six load-bearing corrections to the Wave 12 event framework. See [docs-internal/event_framework.md → Wave 13 fixes](./docs-internal/event_framework.md#wave-13-fixes) for the full breakdown. Key invariants to remember when working on canary triggers: (1) `register_canary_trigger_type(node_type, cloudevent_type)` requires the CloudEvents reverse-DNS string as second arg — must match the producer's `WorkflowEvent.type` exactly or the deployment manager's `EventType` SA won't match `dispatch.emit`'s Visibility query (silent firing failure). Diverging re-registration raises `ValueError` so plugin upgrades surface loudly. (2) Plugin `_events.py` for canary-registered triggers is canary-only — no `event_waiter.dispatch` / `send_custom_event` calls (those have zero consumers in canary-on mode). `dispatch.emit(envelope, wire_routing_key=...)` handles BOTH Temporal Signal fan-out AND in-process WS broadcast. (3) `TriggerListenerWorkflow` + `PollingTriggerWorkflow` call `broadcast_trigger_status_activity` before/after each child spawn for firing-pulse UX (matches legacy `triggers.py` collector/processor). (4) `PollingTriggerNode.as_poll_activity` returns `seen_ids: list(current)` — NOT `list(prior_seen | current)` (the latter grows unboundedly; Gmail at ~100/day hit ~36K entries in a year). The legacy `_build_poll_coroutine` does `seen = set(current)` at end of cycle for the same reason. Visibility-filtered providers (Gmail-unread) re-emit on re-surface, which is correct. (5) Canary trigger output persistence: `MachinaWorkflow.run`'s pre-executed loop schedules `store_node_output_activity` for every firing trigger so `ParameterResolver` can resolve `{{triggerNode.field}}` in downstream nodes (the legacy `_execute_from_trigger` did this via `_store_output(trigger_node_id, "output_0", ...)`; canary skipped it pre-fix). Skips non-firing siblings (`_trigger_output={"not_triggered": True}`). (6) `DeploymentManager.cancel` sweeps stuck node statuses via `_clear_stuck_node_statuses(workflow_id, include_waiting=True)` + emits terminal `update_workflow_status(executing=False, workflow_id=...)`. Without these the FE leaves downstream nodes glowing forever after deployment cancel and the toolbar Start/Stop indicator stays at `executing=True`. The delegation guard inside `_clear_stuck_node_statuses` still protects in-flight fire-and-forget child agents.
- never use emojis in prints
\ No newline at end of file
diff --git a/cli/commands/_temporal_specs.py b/cli/commands/_temporal_specs.py
index c34e6164..1d4ca0a6 100644
--- a/cli/commands/_temporal_specs.py
+++ b/cli/commands/_temporal_specs.py
@@ -1,22 +1,31 @@
"""ServiceSpecs for the Temporal stack — shared by ``start`` and ``dev``.
-Returns the right Temporal ``ServiceSpec`` set based on the
-``TEMPORAL_BACKEND`` env var:
+Returns the right Temporal ``ServiceSpec`` set based on
+``cfg.temporal_backend`` (sourced from ``TEMPORAL_BACKEND`` in
+``.env.template`` -> ``.env`` -> ``os.environ`` by
+:func:`cli.config.load_config`):
- - ``sqlite`` (default, dev) — single spec running ``temporal start-dev``
- against a SQLite file. Same behaviour as before this sprint, but the
- binary now comes from a pooch-managed cache (downloaded on first
- run by the Python backend's ``services.temporal._install``) rather
- than the deprecated ``temporal-server`` npm package.
-
- - ``postgres`` (prod) — two specs running the supervised runtimes:
+ - ``postgres`` (default) — two specs running the supervised runtimes:
``temporal-postgres`` (pgserver) then ``temporal-server`` (Temporal
binary with YAML config). Both wrap ``BaseSupervisor`` singletons
- via ``cli.commands._supervised_runtime``.
+ via ``cli.commands._supervised_runtime``. Data lands under
+ ``DATA_DIR/postgres/``; binaries come from a pooch-managed cache
+ (no system install required).
+
+ - ``sqlite`` — single spec running ``temporal api`` against an
+ in-memory SQLite store. Requires the standalone ``temporal`` CLI on
+ PATH. Lighter dev path; state is lost on restart.
The supervisor's TCP readiness probes (``ready_port``) order the
postgres → temporal startup automatically — Temporal won't start until
Postgres is accepting connections.
+
+No backend default lives in this module: ``cfg.temporal_backend``
+always carries a concrete value loaded from the env files (see
+:mod:`cli.config`). Mismatch with ``core.config.Settings.
+temporal_backend`` would mean the supervisor disagrees with the runtime
+about which spec is active -- the v0.0.79 regression that crashed
+``machina start`` after ``npm install -g machinaos``.
"""
from __future__ import annotations
@@ -26,17 +35,18 @@ from pathlib import Path
from cli.supervisor import RestartPolicy, ServiceSpec
# Backend names — must match ``core.config.Settings.temporal_backend``
-# enum (``"sqlite" | "postgres"``). Read from ``TEMPORAL_BACKEND`` env
-# var at supervisor-build time; the matching Settings field flows
-# through to the runtimes via :class:`core.config.Settings`.
+# enum (``"sqlite" | "postgres"``). The active backend comes from
+# ``cfg.temporal_backend`` (env-file driven via ``cli.config``);
+# no default lives here.
_BACKEND_SQLITE = "sqlite"
_BACKEND_POSTGRES = "postgres"
-_BACKEND_ENV_VAR = "TEMPORAL_BACKEND"
# Readiness-probe windows. ``temporal-server`` may need to download
# its binary on first run (~90 MB tarball); the Postgres init step
-# runs schema migrations once. Override via env if your environment
-# needs different headroom.
+# runs schema migrations once. These are advanced tuning knobs left as
+# pure-process-env overrides (commented hints in ``.env.template``):
+# overriding them is rare and per-environment, not worth a typed
+# Config field.
_PG_READY_TIMEOUT_SECONDS = float(
os.environ.get("TEMPORAL_PG_READY_TIMEOUT_SECONDS", "60"),
)
@@ -44,9 +54,9 @@ _SERVER_READY_TIMEOUT_SECONDS = float(
os.environ.get("TEMPORAL_SERVER_READY_TIMEOUT_SECONDS", "120"),
)
-# Graceful-shutdown grace. Read directly from the same setting the
-# runtime uses (``TEMPORAL_GRACEFUL_SHUTDOWN_SECONDS``) so the
-# supervisor and the Python runtime stay in lockstep.
+# Graceful-shutdown grace. ``TEMPORAL_GRACEFUL_SHUTDOWN_SECONDS`` is
+# declared in ``.env.template`` and pushed into ``os.environ`` by
+# ``cli.config.load_config``, so this read sees the file value.
_GRACE_SECONDS = float(
os.environ.get("TEMPORAL_GRACEFUL_SHUTDOWN_SECONDS", "30"),
)
@@ -66,29 +76,29 @@ def _runtime_argv(factory_path: str) -> list[str]:
def temporal_specs(root: Path, cfg) -> list[ServiceSpec]:
"""Return the Temporal ServiceSpec list for the current backend.
- ``cfg`` is ``cli.config.Config``; ``cfg.temporal_port`` drives
- the gRPC readiness probe regardless of backend (same port for
- both ``sqlite`` and ``postgres`` paths).
+ ``cfg`` is :class:`cli.config.Config`; ``cfg.temporal_backend``
+ picks the spec set and ``cfg.temporal_port`` drives the gRPC
+ readiness probe (same port for both ``sqlite`` and ``postgres``).
"""
- backend = os.environ.get(_BACKEND_ENV_VAR, _BACKEND_SQLITE).strip().lower()
+ backend = cfg.temporal_backend.strip().lower()
server_dir = root / "server"
if backend == _BACKEND_POSTGRES:
return [
ServiceSpec(
- name="temporal-postgres",
+ name="postgres",
argv=_runtime_argv("services.temporal._runtime:get_postgres_runtime"),
cwd=server_dir,
# pgserver picks a dynamic port; skip the TCP probe here
- # and rely on the temporal-server spec to gate readiness
- # via gRPC (it can't connect until Postgres accepts).
+ # and rely on the temporal spec to gate readiness via
+ # gRPC (it can't connect until Postgres accepts).
ready_port=None,
ready_timeout=_PG_READY_TIMEOUT_SECONDS,
restart=RestartPolicy.ON_CRASH,
terminate_grace_seconds=_GRACE_SECONDS,
),
ServiceSpec(
- name="temporal-server",
+ name="temporal",
argv=_runtime_argv(
"services.temporal._runtime:get_temporal_server_runtime",
),
@@ -100,15 +110,23 @@ def temporal_specs(root: Path, cfg) -> list[ServiceSpec]:
),
]
- # sqlite (default) — single spec wrapping the binary in
- # ``temporal api`` (the bundled SQLite dev server).
+ # sqlite — single spec wrapping the pooch-installed ``temporal``
+ # CLI's ``server start-dev`` subcommand against a SQLite db at
+ # ``settings.temporal_sqlite_path``. Same supervised-runtime shim
+ # as the postgres backend; ``TemporalServerRuntime`` switches on
+ # ``settings.temporal_backend`` to pick the right binary + argv
+ # (see ``server/services/temporal/_runtime.py``).
return [
ServiceSpec(
name="temporal",
- argv=["temporal", "api"],
- cwd=root,
+ argv=_runtime_argv(
+ "services.temporal._runtime:get_temporal_server_runtime",
+ ),
+ cwd=server_dir,
ready_port=cfg.temporal_port,
+ ready_timeout=_SERVER_READY_TIMEOUT_SECONDS,
restart=RestartPolicy.ON_CRASH,
+ terminate_grace_seconds=_GRACE_SECONDS,
),
]
diff --git a/cli/commands/build.py b/cli/commands/build.py
index 238a587e..485c5806 100644
--- a/cli/commands/build.py
+++ b/cli/commands/build.py
@@ -89,25 +89,6 @@ def _ensure_uv(python_cmd: str) -> str:
return version
-def _ensure_temporal() -> None:
- """Ensure the ``temporal`` CLI (from npm package ``temporal-server``) is on PATH."""
- version = capture(["temporal", "--version"])
- if version:
- console.print(f" temporal: {version}")
- return
- console.print(" temporal: not found, installing globally...")
- rc = run(["npm", "install", "-g", "temporal-server"], check=False)
- if rc != 0:
- console.print(
- " [yellow]Warning: temporal install failed. "
- "Distributed execution unavailable.[/]"
- )
- return
- version = capture(["temporal", "--version"])
- if version:
- console.print(f" temporal: {version}")
-
-
# ---------------------------------------------------------------- build
def build_command() -> None:
@@ -125,6 +106,10 @@ def build_command() -> None:
return
# ---- toolchain ---------------------------------------------------
+ # Temporal binaries are NOT a build-time dep: the supervised
+ # ``TemporalServerRuntime`` (server/services/temporal/_runtime.py)
+ # downloads ``temporal`` / ``temporal-server`` / ``temporal-sql-tool``
+ # via pooch on first boot. No system install / npm shim required.
console.print("[bold]Checking dependencies...[/]\n")
node_version = capture(["node", "--version"])
console.print(f" Node.js: {node_version or '[red]not found[/]'}")
@@ -144,7 +129,6 @@ def build_command() -> None:
raise typer.Exit(code=1)
_ensure_uv(python_cmd)
- _ensure_temporal()
console.print("\n[green]All dependencies ready.[/]\n")
diff --git a/cli/commands/start.py b/cli/commands/start.py
index 35109c6c..e2e5c23e 100644
--- a/cli/commands/start.py
+++ b/cli/commands/start.py
@@ -25,7 +25,6 @@ from cli.config import load_config
from cli.platform_ import IS_WINDOWS, IS_WSL, platform_name, project_root
from cli.buildenv import validate_build, venv_python
from cli.ports import kill_port
-from cli.run import which_argv
from cli.supervisor import Manager, ServiceSpec
from cli.commands._temporal_specs import temporal_specs
@@ -69,18 +68,14 @@ def _sqlalchemy_preflight(root: Path) -> None:
)
-def _temporal_running() -> bool:
- """Check whether the temporal server is already up via the ``temporal`` CLI."""
- try:
- out = subprocess.run(
- which_argv(["temporal", "status"]),
- capture_output=True,
- text=True,
- timeout=5,
- )
- return any(token in out.stdout for token in ("running", "UP", "up"))
- except (FileNotFoundError, subprocess.TimeoutExpired):
- return False
+def _temporal_running(cfg) -> bool:
+ """Check whether a Temporal frontend is already listening on the
+ configured gRPC port. TCP probe instead of spawning a CLI -- works
+ without the legacy ``temporal-server`` npm wrapper (we install our
+ own binary via pooch at first boot of the supervised
+ ``TemporalServerRuntime``)."""
+ from cli.tcp import probe_tcp_port_sync
+ return probe_tcp_port_sync(cfg.temporal_port)
def _read_version(root: Path) -> str:
@@ -133,7 +128,7 @@ def start_command() -> None:
validate_build(root, require_client_dist=True)
_sqlalchemy_preflight(root)
- temporal_running = _temporal_running()
+ temporal_running = _temporal_running(cfg)
if temporal_running:
console.print("[dim]Temporal already running, skipping[/]")
diff --git a/cli/config.py b/cli/config.py
index 6506c78c..799ce970 100644
--- a/cli/config.py
+++ b/cli/config.py
@@ -1,9 +1,25 @@
-"""Project configuration -- ports + flags from ``.env``.
-
-Stdlib-only env-var loader. ``.env`` is parsed as plain ``KEY=VALUE``
-lines (with ``#`` comments and surrounding ``"`` / ``'`` stripped); the
-backend has python-dotenv if full shell semantics are needed. Process
-env wins over file env, matching the prior pydantic-settings behaviour.
+"""Project configuration -- typed view over ``.env`` / ``.env.template``.
+
+Stdlib-only env-var loader. ``.env.template`` ships with every install
+and is the canonical source of defaults; ``.env`` (created by
+``scripts/postinstall.js`` on ``npm install -g machinaos``) layers user
+overrides on top; ``os.environ`` wins over both at the call site (so
+``TEMPORAL_BACKEND=foo machina start`` still works).
+
+Merged values are pushed into ``os.environ`` so downstream consumers
+that read ``os.environ`` directly (e.g.
+:mod:`cli.commands._temporal_specs` deciding which Temporal ServiceSpec
+set to build) see the same view as this module. Without that push the
+supervisor's environment lookups would return ``None`` even after
+``postinstall.js`` copied ``.env.template`` to ``.env`` -- the bug that
+made v0.0.79's ``npm install -g machinaos && machina start`` path
+silently fall back to the sqlite Temporal spec instead of the
+``postgres`` default declared in ``.env.template``.
+
+No Python-side hardcoded defaults: ``.env.template`` is the single
+source of truth. If the template is missing (broken install), we error
+out loudly with a remediation hint rather than silently substituting
+duplicate hardcoded values that drift from the shipped template.
"""
from __future__ import annotations
@@ -32,30 +48,38 @@ def _load_env_file(path: Path) -> dict[str, str]:
return out
-def _int(env: dict[str, str], key: str, default: int) -> int:
- try:
- return int(env.get(key, default))
- except (ValueError, TypeError):
- return default
-
-
-def _bool(env: dict[str, str], key: str, default: bool) -> bool:
+def _require(env: dict[str, str], key: str) -> str:
raw = env.get(key)
if raw is None:
- return default
- return raw.strip().lower() in ("1", "true", "yes", "on")
+ raise KeyError(
+ f"{key} missing from .env / .env.template -- "
+ "reinstall (``npm install -g machinaos``) or restore the template."
+ )
+ return raw
+
+
+def _int(env: dict[str, str], key: str) -> int:
+ return int(_require(env, key))
+
+
+def _bool(env: dict[str, str], key: str) -> bool:
+ return _require(env, key).strip().lower() in ("1", "true", "yes", "on")
@dataclass(frozen=True)
class Config:
- """Typed view over ``.env``; defaults match the JS ``loadEnvConfig``."""
-
- client_port: int = 3000
- backend_port: int = 3010
- whatsapp_port: int = 9400
- nodejs_port: int = 3020
- temporal_address: str = "localhost:7233"
- temporal_enabled: bool = True
+ """Typed view over the merged ``.env.template`` + ``.env`` + process
+ env. No field defaults -- every value originates in the env files so
+ the published shape matches what users see (and can edit) in their
+ ``.env``."""
+
+ client_port: int
+ backend_port: int
+ whatsapp_port: int
+ nodejs_port: int
+ temporal_address: str
+ temporal_enabled: bool
+ temporal_backend: str
@property
def temporal_port(self) -> int:
@@ -77,17 +101,45 @@ class Config:
@lru_cache(maxsize=1)
def load_config(root: Path | None = None) -> Config:
+ """Load + merge ``.env.template`` (baseline) and ``.env`` (overrides).
+
+ Precedence (low -> high): ``.env.template`` < ``.env`` < ``os.environ``.
+
+ Merged file values are pushed into ``os.environ`` so any downstream
+ ``os.environ.get(...)`` lookup sees the same view -- without
+ overwriting existing process env vars (matches python-dotenv's
+ default ``override=False`` semantics).
+ """
root = root or project_root()
- primary = root / ".env"
- fallback = root / ".env.template"
- file_env = _load_env_file(primary if primary.exists() else fallback)
- env = {**file_env, **os.environ}
+ template = root / ".env.template"
+ user = root / ".env"
+
+ if not template.exists():
+ raise FileNotFoundError(
+ f".env.template not found at {template}. "
+ "Reinstall (``npm install -g machinaos``) or restore the template "
+ "from the source tree."
+ )
+
+ # Layer: .env.template (canonical defaults) <- .env (user override)
+ merged: dict[str, str] = _load_env_file(template)
+ merged.update(_load_env_file(user))
+
+ # Push file values into the process environment so downstream
+ # ``os.environ.get(...)`` reads see them. Process env still wins --
+ # only keys missing from the process environment are inserted.
+ for key, value in merged.items():
+ os.environ.setdefault(key, value)
+
+ # Final view layers process env on top of file env.
+ env = {**merged, **os.environ}
return Config(
- client_port=_int(env, "VITE_CLIENT_PORT", 3000),
- backend_port=_int(env, "PYTHON_BACKEND_PORT", 3010),
- whatsapp_port=_int(env, "WHATSAPP_RPC_PORT", 9400),
- nodejs_port=_int(env, "NODEJS_EXECUTOR_PORT", 3020),
- temporal_address=env.get("TEMPORAL_SERVER_ADDRESS", "localhost:7233"),
- temporal_enabled=_bool(env, "TEMPORAL_ENABLED", True),
+ client_port=_int(env, "VITE_CLIENT_PORT"),
+ backend_port=_int(env, "PYTHON_BACKEND_PORT"),
+ whatsapp_port=_int(env, "WHATSAPP_RPC_PORT"),
+ nodejs_port=_int(env, "NODEJS_EXECUTOR_PORT"),
+ temporal_address=_require(env, "TEMPORAL_SERVER_ADDRESS"),
+ temporal_enabled=_bool(env, "TEMPORAL_ENABLED"),
+ temporal_backend=_require(env, "TEMPORAL_BACKEND"),
)
diff --git a/cli/tests/test_build_compile_pipeline.py b/cli/tests/test_build_compile_pipeline.py
index 27cdcd5a..4d3b0ac2 100644
--- a/cli/tests/test_build_compile_pipeline.py
+++ b/cli/tests/test_build_compile_pipeline.py
@@ -63,11 +63,13 @@ def _run_build_capture_invocations(tmp_path: Path) -> list[dict]:
return "Python 3.12.5"
return "v1.0.0"
+ # ``_ensure_temporal`` was removed -- the supervised
+ # ``TemporalServerRuntime`` downloads the binary via pooch at first
+ # boot, so the build command no longer touches Temporal at all.
with patch.object(build, "run", side_effect=fake_run), \
patch.object(build, "capture", side_effect=fake_capture), \
patch.object(build, "_which_python", return_value="python"), \
patch.object(build, "_ensure_uv"), \
- patch.object(build, "_ensure_temporal"), \
patch.object(build, "project_root", return_value=tmp_path):
build.build_command()
diff --git a/cli/tests/test_dev.py b/cli/tests/test_dev.py
index c5390a10..bca1043d 100644
--- a/cli/tests/test_dev.py
+++ b/cli/tests/test_dev.py
@@ -5,11 +5,15 @@ from __future__ import annotations
from pathlib import Path
from cli.commands import dev
-from cli.config import Config
+from cli.config import Config, load_config
def _cfg() -> Config:
- return Config()
+ # Use the real env-file loader so test config mirrors production
+ # behaviour (``.env.template`` -> ``.env`` -> ``os.environ``).
+ # No hardcoded values: ``.env.template`` is the single source of
+ # truth, same as ``cli.commands.start``/``dev`` at runtime.
+ return load_config()
def test_has_vite_false_when_missing(tmp_path: Path):
diff --git a/cli/tests/test_start.py b/cli/tests/test_start.py
index c761442f..746d2232 100644
--- a/cli/tests/test_start.py
+++ b/cli/tests/test_start.py
@@ -7,11 +7,15 @@ from unittest.mock import patch
from cli import buildenv
from cli.commands import start
-from cli.config import Config
+from cli.config import Config, load_config
def _cfg() -> Config:
- return Config()
+ # Use the real env-file loader so test config mirrors production
+ # behaviour (``.env.template`` -> ``.env`` -> ``os.environ``).
+ # No hardcoded values: ``.env.template`` is the single source of
+ # truth, same as ``cli.commands.start`` at runtime.
+ return load_config()
def test_venv_python_returns_none_when_missing(tmp_path: Path):
@@ -32,9 +36,13 @@ def test_venv_python_finds_posix_layout(tmp_path: Path):
assert buildenv.venv_python(tmp_path) == posix_py
-def test_temporal_running_false_when_cli_missing():
- with patch.object(start.subprocess, "run", side_effect=FileNotFoundError):
- assert start._temporal_running() is False
+def test_temporal_running_false_when_port_closed():
+ # TCP-probe based check: returns False when nothing is listening on
+ # the configured gRPC port. Mock the probe directly instead of the
+ # legacy subprocess-spawn implementation.
+ cfg = _cfg()
+ with patch("cli.tcp.probe_tcp_port_sync", return_value=False):
+ assert start._temporal_running(cfg) is False
def test_build_specs_skips_temporal_when_already_running(tmp_path: Path):
@@ -46,13 +54,16 @@ def test_build_specs_skips_temporal_when_already_running(tmp_path: Path):
def test_build_specs_includes_temporal_when_not_running(tmp_path: Path):
cfg = _cfg()
specs = start._build_specs(tmp_path, cfg, temporal_running=False)
- assert {s.name for s in specs} == {"client", "server", "temporal"}
+ names = {s.name for s in specs}
+ # Backend-agnostic: sqlite yields {"client", "server", "temporal"};
+ # postgres yields {"client", "server", "postgres", "temporal"}.
+ assert {"client", "server", "temporal"} <= names
def test_build_specs_assigns_ready_ports(tmp_path: Path):
cfg = _cfg()
specs = start._build_specs(tmp_path, cfg, temporal_running=False)
by_name = {s.name: s for s in specs}
- assert by_name["client"].ready_port == 3000
- assert by_name["server"].ready_port == 3010
- assert by_name["temporal"].ready_port == 7233
+ assert by_name["client"].ready_port == cfg.client_port
+ assert by_name["server"].ready_port == cfg.backend_port
+ assert by_name["temporal"].ready_port == cfg.temporal_port
diff --git a/server/core/config.py b/server/core/config.py
index bc7ce0bf..2b602f42 100644
--- a/server/core/config.py
+++ b/server/core/config.py
@@ -144,6 +144,26 @@ class Settings(BaseSettings):
temporal_postgres_dsn: Optional[str] = Field(
default=None, env="TEMPORAL_POSTGRES_DSN",
)
+ # On-disk locations for Temporal persistence. Both resolve relative
+ # to ``DATA_DIR`` (cross-platform via ``Settings._resolve_under_data``);
+ # absolute paths are used verbatim. ``temporal_sqlite_path`` is the
+ # SQLite database file the dev server opens via ``temporal server
+ # start-dev --db-filename ...``. ``temporal_postgres_subdir`` is the
+ # pgserver data directory (where PostgreSQL writes its own files
+ # before Temporal touches them). ``temporal_server_subdir`` is the
+ # cwd for the postgres-backed ``temporal-server`` process (config
+ # YAML lands there). All three are env-driven so test harnesses and
+ # alternative installs can point them wherever needed without code
+ # changes.
+ temporal_sqlite_path: str = Field(
+ default="temporal/temporal.db", env="TEMPORAL_SQLITE_PATH",
+ )
+ temporal_postgres_subdir: str = Field(
+ default="postgres", env="TEMPORAL_POSTGRES_SUBDIR",
+ )
+ temporal_server_subdir: str = Field(
+ default="_temporal", env="TEMPORAL_SERVER_SUBDIR",
+ )
# API Keys (all optional, injected at runtime)
google_maps_api_key: Optional[str] = Field(default=None, env="GOOGLE_MAPS_API_KEY")
diff --git a/server/services/temporal/_install.py b/server/services/temporal/_install.py
index 32909195..7b65376f 100644
--- a/server/services/temporal/_install.py
+++ b/server/services/temporal/_install.py
@@ -71,10 +71,17 @@ _CHECKSUMS_BY_VERSION: dict[str, dict[str, str]] = {
}
# Names of the binaries we extract from each release tarball. The
-# Temporal release ships several; these are the two MachinaOS actually
-# invokes (server + schema migrator). Adding ``tctl`` here would also
-# extract it.
-_BINARY_NAMES: tuple[str, ...] = ("temporal-server", "temporal-sql-tool")
+# Temporal release ships several; we extract the three MachinaOS
+# actually invokes:
+# - ``temporal`` — standalone CLI. Powers the sqlite-backend dev
+# server (``temporal server start-dev``) and any
+# ad-hoc workflow / operator commands.
+# - ``temporal-server`` — full cluster binary used by the postgres
+# backend with a YAML config.
+# - ``temporal-sql-tool`` — schema migrator invoked once at boot to
+# create + update the Postgres databases.
+# Adding ``tctl`` here would also extract it.
+_BINARY_NAMES: tuple[str, ...] = ("temporal", "temporal-server", "temporal-sql-tool")
# pooch cache namespace. Re-used across versions; pooch's own cache
# layout includes the asset URL so versions don't collide.
diff --git a/server/services/temporal/_runtime.py b/server/services/temporal/_runtime.py
index 4f8129f1..1d5d5a94 100644
--- a/server/services/temporal/_runtime.py
+++ b/server/services/temporal/_runtime.py
@@ -72,7 +72,7 @@ class PostgresRuntime(BaseSupervisor):
binary.
"""
- name = "temporal-postgres"
+ name = "postgres"
def __init__(self, settings: Optional[Settings] = None) -> None:
super().__init__()
@@ -86,12 +86,18 @@ class PostgresRuntime(BaseSupervisor):
def is_running(self) -> bool:
return self._pg is not None
+ @property
+ def _data_dir(self) -> Path:
+ """Env-driven pgserver data dir (``TEMPORAL_POSTGRES_SUBDIR``),
+ resolved relative to ``DATA_DIR`` unless absolute."""
+ return Path(self.settings._resolve_under_data(self.settings.temporal_postgres_subdir))
+
async def _do_start(self) -> None:
# Lazy import: pgserver pulls a heavy native dep tree; only
# load it when the Postgres backend is actually selected.
import pgserver
- data_dir = Path(self.settings.data_dir) / "postgres"
+ data_dir = self._data_dir
data_dir.mkdir(parents=True, exist_ok=True)
def _start_sync() -> Any:
@@ -127,7 +133,7 @@ class PostgresRuntime(BaseSupervisor):
return {
"uri": self.uri,
"port": self.port,
- "data_dir": str(Path(self.settings.data_dir) / "postgres"),
+ "data_dir": str(self._data_dir),
}
# ---- public read-only properties -------------------------------------
@@ -146,20 +152,39 @@ class PostgresRuntime(BaseSupervisor):
class TemporalServerRuntime(BaseProcessSupervisor):
- """Temporal server binary, supervised via BaseProcessSupervisor.
-
- The binary lands on disk via :func:`services.temporal._install.ensure_temporal_binaries`
- (pooch-cached); the YAML config is rendered by
- :func:`services.temporal._config.render_temporal_config` pointing at
- the running :class:`PostgresRuntime` instance.
-
- BaseProcessSupervisor handles all signal, restart, and tree-kill
- semantics — we only override the four required methods plus
- ``_pre_spawn`` (download + bootstrap) and ``health_check``
- (gRPC port probe).
+ """Temporal server, supervised via BaseProcessSupervisor.
+
+ Single runtime class for both backends. The only difference is
+ which binary + argv we spawn and whether we run the Postgres
+ schema-bootstrap step; everything else (signal / restart /
+ tree-kill / health probe) is shared. Backend selected by
+ ``settings.temporal_backend``:
+
+ - ``postgres`` — spawns ``temporal-server start --config <yaml>``
+ against a running :class:`PostgresRuntime` instance. YAML config
+ is rendered by
+ :func:`services.temporal._config.render_temporal_config`;
+ schemas are migrated via
+ :func:`services.temporal._config.bootstrap_temporal_schemas`.
+
+ - ``sqlite`` — spawns ``temporal server start-dev --db-filename
+ <path>`` against a SQLite file. Path comes from
+ ``settings.temporal_sqlite_path`` (env-driven).
+
+ Both binaries (``temporal``, ``temporal-server``,
+ ``temporal-sql-tool``) ship in the same pooch-cached tarball
+ extracted by
+ :func:`services.temporal._install.ensure_temporal_binaries`, so
+ swapping backends costs no extra download.
+
+ All on-disk locations come from env-driven settings (no hardcoded
+ subdirs in this module): ``settings.temporal_sqlite_path``,
+ ``settings.temporal_postgres_subdir``,
+ ``settings.temporal_server_subdir`` — all resolved relative to
+ ``settings.data_dir`` unless already absolute.
"""
- name = "temporal-server"
+ name = "temporal"
pipe_streams = True
graceful_shutdown = sys.platform == "win32"
@@ -174,63 +199,119 @@ class TemporalServerRuntime(BaseProcessSupervisor):
self.settings = settings
# SIGTERM grace = settings.temporal_graceful_shutdown_seconds —
# same knob already documented for the embedded Temporal worker.
- # Reusing it avoids inventing a parallel ``temporal_server_grace_seconds``.
self.terminate_grace_seconds = float(
settings.temporal_graceful_shutdown_seconds,
)
- self._postgres = postgres or PostgresRuntime.get_instance(settings)
+ # Postgres runtime is only consulted when ``backend == "postgres"``;
+ # lazy-bound there so the sqlite path doesn't spin up an unused
+ # PostgresRuntime singleton or pull pgserver imports.
+ self._postgres = postgres
self._binaries: Optional[dict[str, Path]] = None
self._config_path: Optional[Path] = None
+ @property
+ def backend(self) -> str:
+ return self.settings.temporal_backend.strip().lower()
+
+ @property
+ def _sqlite_path(self) -> Path:
+ """Env-driven SQLite db path (``TEMPORAL_SQLITE_PATH``),
+ resolved relative to ``DATA_DIR`` unless absolute."""
+ return Path(self.settings._resolve_under_data(self.settings.temporal_sqlite_path))
+
+ @property
+ def _server_cwd(self) -> Path:
+ """Env-driven cwd subdir for the temporal-server process
+ (``TEMPORAL_SERVER_SUBDIR``)."""
+ return Path(self.settings._resolve_under_data(self.settings.temporal_server_subdir))
+
# ---- BaseProcessSupervisor overrides ---------------------------------
async def _pre_spawn(self) -> None:
from services.temporal._install import ensure_temporal_binaries
- from services.temporal._config import (
- bootstrap_temporal_schemas,
- render_temporal_config,
- )
- if self._postgres.uri is None:
- raise RuntimeError(
- f"[{self.label}] Postgres runtime not started; "
- "schedule the postgres ServiceSpec before this one"
- )
-
- # 1. Download / cache temporal-server + temporal-sql-tool.
- # Version pin lives in settings.temporal_binary_version.
+ # 1. Download / cache Temporal binaries (both backends share
+ # the same tarball; pooch deduplicates across calls).
self._binaries = await ensure_temporal_binaries(self.settings)
- # 2. Idempotent schema bootstrap.
- await bootstrap_temporal_schemas(
- sql_tool=self._binaries["temporal-sql-tool"],
- postgres_uri=self._postgres.uri,
- binary_path=self._binaries["temporal-server"],
- )
+ if self.backend == "postgres":
+ from services.temporal._config import (
+ bootstrap_temporal_schemas,
+ render_temporal_config,
+ )
- # 3. Render YAML config pointing at the Postgres URI.
- self._config_path = render_temporal_config(
- settings=self.settings,
- postgres_uri=self._postgres.uri,
- )
+ if self._postgres is None:
+ self._postgres = PostgresRuntime.get_instance(self.settings)
+ if self._postgres.uri is None:
+ raise RuntimeError(
+ f"[{self.label}] Postgres runtime not started; "
+ "schedule the postgres ServiceSpec before this one"
+ )
+
+ # Idempotent schema bootstrap.
+ await bootstrap_temporal_schemas(
+ sql_tool=self._binaries["temporal-sql-tool"],
+ postgres_uri=self._postgres.uri,
+ binary_path=self._binaries["temporal-server"],
+ )
+
+ # Render YAML config pointing at the Postgres URI.
+ self._config_path = render_temporal_config(
+ settings=self.settings,
+ postgres_uri=self._postgres.uri,
+ )
+ else:
+ # sqlite backend — ensure the parent dir for the SQLite
+ # file exists before ``temporal server start-dev`` opens it.
+ self._sqlite_path.parent.mkdir(parents=True, exist_ok=True)
def binary_path(self) -> Path:
+ # Pre-``_pre_spawn`` placeholders for any existence check
+ # BaseProcessSupervisor may run; the real paths are populated
+ # once pooch resolves the cache.
+ binary = "temporal-server" if self.backend == "postgres" else "temporal"
if self._binaries is None:
- # Called before _pre_spawn — e.g. if BaseProcessSupervisor
- # tries to validate the binary exists. Return a placeholder;
- # _pre_spawn populates the real path before the existence
- # check in _do_start runs.
- return Path("temporal-server")
- return self._binaries["temporal-server"]
+ return Path(binary)
+ return self._binaries[binary]
def argv(self) -> list[str]:
- return [str(self.binary_path()), "start", "--config", str(self._config_path)]
+ if self.backend == "postgres":
+ return [
+ str(self.binary_path()),
+ "start", "--config", str(self._config_path),
+ ]
+
+ # ``temporal server start-dev`` is the official subcommand for
+ # the SQLite-backed dev server. Flags documented at
+ # https://docs.temporal.io/cli/server (subset we use):
+ # --port frontend gRPC port (gates ready-probe)
+ # --db-filename SQLite file (omit for in-memory)
+ # --metrics-port 0 disables the Prometheus endpoint
+ # --log-level warn keeps the supervisor log readable
+ # --ip 127.0.0.1 when TEMPORAL_BIND_LOCAL_ONLY=true
+ # --namespace default namespace bootstrapped at start
+ ip = "127.0.0.1" if self.settings.temporal_bind_local_only else "0.0.0.0"
+ return [
+ str(self.binary_path()), "server", "start-dev",
+ "--port", str(self.settings.temporal_frontend_grpc_port),
+ "--db-filename", str(self._sqlite_path),
+ "--metrics-port", "0",
+ "--log-level", "warn",
+ "--ip", ip,
+ "--namespace", self.settings.temporal_namespace,
+ ]
def cwd(self) -> Path:
- return Path(self.settings.data_dir) / "_temporal"
+ if self.backend == "postgres":
+ return self._server_cwd
+ # sqlite — cwd is the parent of the SQLite file so any default
+ # output / log files land alongside the db rather than in the
+ # supervisor's working directory.
+ return self._sqlite_path.parent
def env(self) -> dict[str, str]:
- # Temporal reads everything from YAML; inherit parent env only.
+ # Temporal reads everything from YAML (postgres) or argv flags
+ # (sqlite); inherit parent env only.
return {**os.environ}
async def health_check(self) -> bool:
@@ -243,12 +324,19 @@ class TemporalServerRuntime(BaseProcessSupervisor):
def _extra_status(self) -> dict[str, Any]:
base = super()._extra_status()
- return {
+ extra: dict[str, Any] = {
**base,
+ "backend": self.backend,
"grpc_port": self.settings.temporal_frontend_grpc_port,
- "config_path": str(self._config_path) if self._config_path else None,
"binary_version": self.settings.temporal_binary_version,
}
+ if self.backend == "postgres":
+ extra["config_path"] = (
+ str(self._config_path) if self._config_path else None
+ )
+ else:
+ extra["sqlite_path"] = str(self._sqlite_path)
+ return extra
# ---- module-level singleton accessors -----------------------------------