Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
perf(build): compile pipeline — tsgo type-check, esbuild sidecar, manualChunks, compileall
trohitg0b45fb1a1695
diff --git a/.github/workflows/predeploy.yml b/.github/workflows/predeploy.ymlindex 0a61c812..8cf096c6 100644--- a/.github/workflows/predeploy.yml+++ b/.github/workflows/predeploy.yml@@ -26,7 +26,10 @@ jobs:run: pnpm --filter react-flow-client run lint- name: TypeScript check- run: pnpm --filter react-flow-client exec tsc --noEmit+ # Uses tsgo (`@typescript/native-preview`, pinned in client/package.json).+ # ~5x faster than tsc on this codebase; both pass identically.+ # `typecheck:tsc` script remains available as a fallback if tsgo regresses.+ run: pnpm --filter react-flow-client run typecheck# Test build and start from source (cross-platform)test-build-start:diff --git a/client/package.json b/client/package.jsonindex febc2776..935c8647 100644--- a/client/package.json+++ b/client/package.json@@ -8,6 +8,8 @@"build": "vite build","lint": "eslint .","preview": "vite preview",+ "typecheck": "tsgo --noEmit",+ "typecheck:tsc": "tsc --noEmit","test": "vitest run","test:watch": "vitest","test:coverage": "vitest run --coverage",@@ -84,6 +86,7 @@"@testing-library/jest-dom": "^6.9.1","@testing-library/react": "^16.3.2","@testing-library/user-event": "^14.5.2",+ "@typescript/native-preview": "7.0.0-dev.20260504.1","@typescript-eslint/parser": "^8.54.0","@vitest/coverage-v8": "^2.1.9","babel-plugin-react-compiler": "19.1.0-rc.3",diff --git a/client/src/test/setup.ts b/client/src/test/setup.tsindex afe372e0..392fec6a 100644--- a/client/src/test/setup.ts+++ b/client/src/test/setup.ts@@ -45,6 +45,7 @@ if (typeof globalThis.IntersectionObserver === 'undefined') {takeRecords() { return []; }root = null;rootMargin = '';+ scrollMargin = '';thresholds = [];}// TS infers the assignment is fine now (target widened); kept onlydiff --git a/client/src/vite-env.d.ts b/client/src/vite-env.d.tsindex 476f613f..ae8e86db 100644--- a/client/src/vite-env.d.ts+++ b/client/src/vite-env.d.ts@@ -1,4 +1,11 @@/// <reference types="vite/client" />+/// <reference types="google.maps" />+// `google.maps` namespace is exposed via @types/google.maps. tsc 5.x+// auto-loads all @types/* packages, but tsgo (TypeScript 7 native+// preview) does not pick the namespace up reliably under pnpm's+// symlinked node_modules layout — the explicit reference here is the+// canonical fix per https://developers.google.com/maps/documentation/javascript/using-typescript+// and works for both compilers.interface ImportMetaEnv {readonly VITE_ANDROID_RELAY_URL: string;diff --git a/client/tsconfig.json b/client/tsconfig.jsonindex 04eb5481..b28d8cba 100644--- a/client/tsconfig.json+++ b/client/tsconfig.json@@ -19,7 +19,7 @@"noUnusedLocals": true,"noUnusedParameters": true,"noFallthroughCasesInSwitch": true,- "baseUrl": ".",+ "typeRoots": ["./node_modules/@types"],"paths": {"@/*": ["./src/*"]}diff --git a/client/vite.config.js b/client/vite.config.jsindex a93076ba..543e1ff4 100644--- a/client/vite.config.js+++ b/client/vite.config.js@@ -94,12 +94,72 @@ export default defineConfig(({ mode }) => {host: true},build: {- // antd + reactflow are large libraries - this is expected- chunkSizeWarningLimit: 1500,+ // ES2022 unlocks native `findLast`, optional-chaining-assignment,+ // class-fields without polyfills. Browser baseline: Chrome 94+,+ // Firefox 93+, Safari 15.4+ — within React 19 / Tailwind 4's range.+ target: 'es2022',+ // Lowered from 1500 KB once manualChunks splits out the heavy libs.+ // 850 KB covers the largest legitimate chunk today (`vendor-icons`,+ // ~800 KB raw / 210 KB gz from lucide-react + @lobehub/icons brand+ // SVGs that resist tree-shaking). The cap still catches new chunk+ // bloat in the entry / route bundles.+ chunkSizeWarningLimit: 850,// Emit sourcemaps only when running the bundle analyzer so the// visualizer can attribute bytes to source files accurately.// Skipped in normal production builds to keep build time down.sourcemap: analyze,+ rollupOptions: {+ output: {+ // Split heavy npm libs into their own cacheable chunks so the+ // main bundle stays lean and unrelated dependency churn doesn't+ // bust the user's cache. Group rationale:+ // vendor-react — core framework + form runtime+ // vendor-flow — reactflow is huge and only canvas pages use it+ // vendor-radix — accessibility primitives, used widely+ // vendor-icons — lucide + lobehub brand icons+ // vendor-query — TanStack Query + persistence+ // vendor-markdown — markdown rendering stack (chat / docs panels)+ // vendor-misc — small but heavyweight utilities+ // Anything not listed falls into the default route/entry chunks.+ manualChunks: {+ 'vendor-react': [+ 'react',+ 'react-dom',+ 'react-hook-form',+ '@hookform/resolvers',+ ],+ 'vendor-flow': ['reactflow'],+ 'vendor-radix': [+ 'radix-ui',+ '@radix-ui/react-collapsible',+ '@radix-ui/react-dialog',+ '@radix-ui/react-slot',+ ],+ 'vendor-icons': ['lucide-react', '@lobehub/icons'],+ 'vendor-query': [+ '@tanstack/react-query',+ '@tanstack/query-sync-storage-persister',+ '@tanstack/react-query-persist-client',+ '@lukemorales/query-key-factory',+ ],+ 'vendor-markdown': [+ 'react-markdown',+ 'remark-gfm',+ 'remark-breaks',+ 'prismjs',+ 'react-simple-code-editor',+ '@uiw/react-json-view',+ ],+ 'vendor-misc': [+ 'idb-keyval',+ 'fuzzysort',+ 'cmdk',+ 'sonner',+ 'qrcode.react',+ ],+ },+ },+ },},}})diff --git a/docs-internal/release_build_pipeline.md b/docs-internal/release_build_pipeline.mdnew file mode 100644index 00000000..9bd848f4--- /dev/null+++ b/docs-internal/release_build_pipeline.md@@ -0,0 +1,100 @@+# Release Build Pipeline++Compile-step plan for the npm distribution `machinaos`. Goal: cut cold-start ~20s further on top of the lazy-LangChain fix already in v0.0.76, shrink the Vite main bundle below 200 KB gz, and drop the `tsx` interpreter cost from the Node.js sidecar.++User scope (confirmed before this work): stay on npm distribution; **no** Nuitka/PyOxidizer standalone binary channel; optimize Vite output too; **skip** the plugin-walker (~11s) and service-status-refresh (~20s) hotspots — those are runtime concerns, not compile-time.++## Tooling++| Layer | Tool | Why |+|---|---|---|+| TypeScript type-check | `@typescript/native-preview` (tsgo) | Microsoft's Go-port of `tsc`, ~10x faster on `--noEmit`. Type-check only — JS emit in tsgo is still preview. Vite/esbuild keep producing the actual JS bundles. Stays in `devDependencies`, never ships to users. |+| Vite output | `manualChunks` + `target: 'es2022'` | Split heavy libs (reactflow, radix-ui, lobehub, react-markdown stack) so the main bundle no longer hits the 1500KB warning ceiling. ES2022 unlocks `findLast` / optional-chaining-assignment without polyfills (Chrome 94+, FF 93+, Safari 15.4+ — within React 19 / Tailwind 4 baseline). |+| Node sidecar | `esbuild` bundle to `dist/index.js`, run via `node` | Drops tsx interpreter startup (~500ms-1s every server boot). `--packages=external` keeps Express in `node_modules/` for patch flow. |+| Python | `python -O -m compileall -q -j 0 server/` | Pre-compile bytecode. CLAUDE.md aspirationally documents this; `machina/commands/build.py` doesn't actually do it today. ~5-10s cold-start gain. `-O` strips asserts + `__debug__` branches. |++## Implementation steps++### 1. tsgo type-check++- `client/package.json` → add `"@typescript/native-preview"` to `devDependencies`, add scripts:+ - `"typecheck": "tsgo --noEmit"`+ - `"typecheck:tsc": "tsc --noEmit"` (fallback during the rollout)+- `.github/workflows/release.yml` → add `pnpm --filter react-flow-client run typecheck` to the predeploy gate before the build job.++### 2. Vite manualChunks + target++- `client/vite.config.js` → extend `build` block:+ - `target: 'es2022'`+ - `chunkSizeWarningLimit: 600` (down from 1500)+ - `rollupOptions.output.manualChunks` mapping:+ - `vendor-react`: `react`, `react-dom`, `react-hook-form`, `@hookform/resolvers`+ - `vendor-flow`: `reactflow`+ - `vendor-radix`: `@radix-ui/*`, `radix-ui`+ - `vendor-icons`: `lucide-react`, `@lobehub/icons`+ - `vendor-query`: `@tanstack/react-query`, `@tanstack/query-sync-storage-persister`, `@tanstack/react-query-persist-client`, `@lukemorales/query-key-factory`+ - `vendor-markdown`: `react-markdown`, `remark-gfm`, `remark-breaks`, `prismjs`, `react-simple-code-editor`, `@uiw/react-json-view`+ - `vendor-misc`: `idb-keyval`, `fuzzysort`, `cmdk`, `sonner`, `qrcode.react`++Keep `sourcemap: analyze` (already correct), keep React Compiler config.++### 3. Node sidecar esbuild bundle++- `server/nodejs/package.json` → add `esbuild` devDep; replace scripts:+ - `"build": "esbuild src/index.ts --bundle --platform=node --target=node22 --format=esm --packages=external --outfile=dist/index.js"`+ - `"start": "node dist/index.js"` (was `tsx src/index.ts`)+ - keep `"dev": "tsx watch src/index.ts"`+- `server/nodejs/.gitignore` → new file: `dist/`++### 4. Python bytecode pre-compile++- `machina/commands/build.py` → after `uv sync`, add:+ ```+ console.log("[5/5] Compiling Python bytecode...")+ run(["uv", "run", "python", "-O", "-m", "compileall", "-q", "-j", "0", "."], cwd=server_dir)+ ```+ Fix the existing mislabeled `[3/4]` and `[4/4]` lines to match the new `/5` total.++The npm tarball still excludes `__pycache__/` per `package.json` `files` (cross-Python-minor pyc fragility) — `compileall` runs on the user's machine via `machina build` or `scripts/install.js` post-install.++### 5. Wire bundle + compileall into install.js++- `scripts/install.js` → after `uv sync`:+ 1. `npm --prefix server/nodejs run build` — produce `dist/index.js`+ 2. `python -O -m compileall -q -j 0 server/` — same as build.py++Idempotent on re-runs (compileall only rewrites stale pyc; esbuild is deterministic).++### 6. Tarball verification++- `npm pack --dry-run` after the change. Confirm `server/nodejs/dist/index.js` is included (existing `server/` glob already covers it). Confirm no `__pycache__/` leakage.++## Critical files++| File | Action |+|---|---|+| `client/package.json` | + tsgo devDep, + typecheck scripts |+| `client/vite.config.js` | + manualChunks, target, lower warning |+| `server/nodejs/package.json` | + esbuild devDep, build script, change start |+| `server/nodejs/.gitignore` | new — ignore `dist/` |+| `machina/commands/build.py` | + compileall step, fix step numbering |+| `scripts/install.js` | + sidecar bundle + compileall calls |+| `.github/workflows/release.yml` | + typecheck gate before build |++## Verification++1. `pnpm --filter react-flow-client run typecheck` → <5s, zero errors.+2. `ANALYZE=1 pnpm --filter react-flow-client run build` → open `client/dist/stats.html`. Expect: no chunk above 600 KB gz, main < 200 KB gz, `vendor-flow` split.+3. `cd server/nodejs && npm run build && node dist/index.js` → starts on :3020 in <100ms.+4. `cd server && uv run python -O -m compileall -q -j 0 .` → `__pycache__/*.opt-1.pyc` present.+5. Cold-start: clean install + `machina start > start.log 2>&1` → `Application startup complete` at ≤+50s (was +66.9s).+6. `npm pack --dry-run` → `server/nodejs/dist/index.js` included; no `__pycache__/`; tarball size ≤ v0.0.76.+7. Smoke: `machina start` → load http://localhost:3000 → run "AI Assistant" example → agent responds.++## Out of scope (future work)++- Nuitka / PyOxidizer standalone binaries (separate release channel, ~1-2 weeks CI matrix).+- Plugin walker lazy-loading (~11s).+- Service-status refresh parallelisation (~20s post-startup-complete).+- mypyc / Cython for hot paths (low ROI — pydantic V2 already Rust, httpx/aiohttp already C).+- swc / stc TypeScript checker (not production-grade in 2026).diff --git a/machina/commands/build.py b/machina/commands/build.pyindex 3492f1a4..42cd6d67 100644--- a/machina/commands/build.py+++ b/machina/commands/build.py@@ -135,28 +135,54 @@ def build_command() -> None:env_path = root / ".env"template_path = root / ".env.template"- # Step markers go through ``console.log`` so each [N/5] line is+ # Step markers go through ``console.log`` so each [N/6] line is# timestamped — diff between consecutive timestamps is the wall-clock# cost of that step, no manual instrumentation needed.if not env_path.exists() and template_path.exists():shutil.copy2(template_path, env_path)- console.log("[0/5] Created .env from template")+ console.log("[0/6] Created .env from template")if not is_postinstall:- console.log("[1/5] Installing dependencies...")+ console.log("[1/6] Installing dependencies...")run(["pnpm", "install"], cwd=root)else:- console.log("[1/5] Dependencies already installed by package manager")+ console.log("[1/6] Dependencies already installed by package manager")- console.log("[2/5] Building client...")+ console.log("[2/6] Building client...")run(["pnpm", "--filter", "react-flow-client", "run", "build"], cwd=root)- console.log("[3/4] Installing Python dependencies...")+ # Pre-bundle the Node.js sidecar (server/nodejs) with esbuild so the+ # production `npm start` runs `node dist/index.js` instead of+ # interpreting `tsx src/index.ts`. Saves ~500ms-1s of cold start+ # whenever the executor is launched. The bundle keeps Express+ # external (it stays in node_modules), so the patch flow is intact.+ console.log("[3/6] Building Node.js sidecar...")+ run(["pnpm", "--filter", "machinaos-nodejs-executor", "run", "build"], cwd=root)++ console.log("[4/6] Installing Python dependencies...")if not (server_dir / ".venv").exists():run(["uv", "venv"], cwd=server_dir)run(["uv", "sync"], cwd=server_dir)- console.log("[4/4] Verifying edgymeow binary...")+ # Pre-compile our Python sources to optimised bytecode (.opt-1.pyc).+ # `-O` strips `assert` statements + `__debug__` branches; `-q`+ # silences per-file output (errors still print); `-j 0` parallelises+ # across all CPU cores. Cuts a measurable few seconds off cold+ # start by avoiding source-to-bytecode work on first import.+ #+ # Scoped to the project's own source dirs — `uv sync` already+ # compiles `.venv/` packages, and `tests/` ships outside the+ # tarball. Compiling `.venv/` is wasted work and fails on cookiecutter+ # template files inside packages like crawlee that aren't real Python.+ console.log("[5/6] Compiling Python bytecode...")+ source_dirs = ["services", "core", "nodes", "routers", "models", "middleware", "main.py", "constants.py"]+ run(+ ["uv", "run", "python", "-O", "-m", "compileall", "-q", "-j", "0", *source_dirs],+ cwd=server_dir,+ check=False, # missing pyc is non-fatal — runtime regenerates as needed+ )++ console.log("[6/6] Verifying edgymeow binary...")bin_name = "edgymeow-server.exe" if sys.platform == "win32" else "edgymeow-server"edgymeow_bin = root / "node_modules" / "edgymeow" / "bin" / bin_nameif edgymeow_bin.exists():diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 99c4cb38..6c4ac4ce 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -201,6 +201,9 @@ importers:'@typescript-eslint/parser':specifier: ^8.54.0+ '@typescript/native-preview':+ specifier: 7.0.0-dev.20260504.1+ version: 7.0.0-dev.20260504.1'@vitest/coverage-v8':specifier: ^2.1.9version: 2.1.9([email protected](@types/[email protected])([email protected])([email protected])([email protected](@types/[email protected])([email protected])))@@ -2723,6 +2726,53 @@ packages:resolution: {integrity: sha512-VFlhGSl4opC0bprJiItPQ1RfUhGDIBokcPwaFH4yiBCaNPeld/9VeXbiPO1cLyorQi1G1vL+ecBk1x8o1axORA==}engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0}+ '@typescript/[email protected]':+ resolution: {integrity: sha512-+Qs1Q7Qxfp11n/hU3pweFU+EQ37FnDsdWOOxb7/vCy8QGBysrLUUYRhQ+GSW3s663oMtN6+9Kf82hk3ZT+kXlg==}+ engines: {node: '>=16.20.0'}+ cpu: [arm64]+ os: [darwin]++ '@typescript/[email protected]':+ resolution: {integrity: sha512-Wr3GWTRiMgibmhe88cjQ612ZyY7sbgsPYEaWKGPUxBaXtMHFIzgTBIoJMuaQqQx4GEJs6AUDyhnIHG1gx4rJjg==}+ engines: {node: '>=16.20.0'}+ cpu: [x64]+ os: [darwin]++ '@typescript/[email protected]':+ resolution: {integrity: sha512-y1Qai5l55Sl+/3B0hyQtvynq//C22BKFH3CfU35fbLYUo4P/ISUycyAbcA+PAPazpDFO3E56I96QUQrbJL2VVA==}+ engines: {node: '>=16.20.0'}+ cpu: [arm64]+ os: [linux]++ '@typescript/[email protected]':+ resolution: {integrity: sha512-s8QkhZe0M4QD2xhK1Xiy2JUQv1AOl8kUg5DLx1G8ws0f1BK/oKyqDNbxhZMGINYLFvkjpr9lOxt7qehSnpJMYQ==}+ engines: {node: '>=16.20.0'}+ cpu: [arm]+ os: [linux]++ '@typescript/[email protected]':+ resolution: {integrity: sha512-ngN3Ie3Vin6pFtqeNywxm86RTxgI0Fo0GZyJ1PxokLES8J3xfMPtMYfv85c/+5uz5+7T+m4LRLyY5IoLY4gtuw==}+ engines: {node: '>=16.20.0'}+ cpu: [x64]+ os: [linux]++ '@typescript/[email protected]':+ resolution: {integrity: sha512-/GZDJN/CsLbqIe7EdWDkXhNX9C41VjemBeUN6+9ckvEFLH8XyKTmXPYikNOn0N819M1KSeNZltplyUslfROOdw==}+ engines: {node: '>=16.20.0'}+ cpu: [arm64]+ os: [win32]++ '@typescript/[email protected]':+ resolution: {integrity: sha512-EYQBdVZq4xIzhTtKxw6wvee9238hEb7XrPG413AEZBD3kcR3qqvPULXsPzQyEpneCReATSaihscP/LfhMQYUmA==}+ engines: {node: '>=16.20.0'}+ cpu: [x64]+ os: [win32]++ '@typescript/[email protected]':+ resolution: {integrity: sha512-bHFGxyIU83qjj6ywn3817A+Ug2ZID0GiBA5WFdbc/T7EjcrKnUUylexq0fU81N/mTbfw3FyP6ZCEdO2Ntcl/VQ==}+ engines: {node: '>=16.20.0'}+ hasBin: true+'@uiw/[email protected]':resolution: {integrity: sha512-botRpQ5AgymYEsqXSdT2/1LefAJEYfMntvdnx1SqhTQCTW9HygeFZXx9inkYqUmiQZ3+0QlZnodjBvwnUfZhVA==}peerDependencies:@@ -9376,6 +9426,37 @@ snapshots:'@typescript-eslint/types': 8.54.0eslint-visitor-keys: 4.2.1+ '@typescript/[email protected]':+ optional: true++ '@typescript/[email protected]':+ optional: true++ '@typescript/[email protected]':+ optional: true++ '@typescript/[email protected]':+ optional: true++ '@typescript/[email protected]':+ optional: true++ '@typescript/[email protected]':+ optional: true++ '@typescript/[email protected]':+ optional: true++ '@typescript/[email protected]':+ optionalDependencies:+ '@typescript/native-preview-darwin-arm64': 7.0.0-dev.20260504.1+ '@typescript/native-preview-darwin-x64': 7.0.0-dev.20260504.1+ '@typescript/native-preview-linux-arm': 7.0.0-dev.20260504.1+ '@typescript/native-preview-linux-arm64': 7.0.0-dev.20260504.1+ '@typescript/native-preview-linux-x64': 7.0.0-dev.20260504.1+ '@typescript/native-preview-win32-arm64': 7.0.0-dev.20260504.1+ '@typescript/native-preview-win32-x64': 7.0.0-dev.20260504.1+'@uiw/[email protected](@babel/[email protected])([email protected]([email protected]))([email protected])':dependencies:'@babel/runtime': 7.28.4diff --git a/scripts/install.js b/scripts/install.jsindex 3af69c5a..5fc531eb 100755--- a/scripts/install.js+++ b/scripts/install.js@@ -159,7 +159,7 @@ try {// Calculate total stepslet totalSteps = 1; // .env alwaysif (!clientDistExists) totalSteps += 2; // client deps + build- totalSteps += 1; // Python deps always+ totalSteps += 2; // Python deps + bytecode compilelet step = 0;// Create .env if needed@@ -198,6 +198,21 @@ try {}run('uv sync', serverDir, 600000); // 10 min timeout+ // Pre-compile our Python sources to optimised bytecode (.opt-1.pyc).+ // `-O` strips assertions and `__debug__` branches; `-q` silences+ // per-file output; `-j 0` parallelises across CPU cores. Scoped to+ // our own source dirs — `uv sync` already compiles `.venv/` and+ // some site-packages contain non-Python template files that would+ // log spurious errors. Failure is non-fatal: the runtime regenerates+ // missing .pyc on first import. Trims a few seconds off cold start.+ step++;+ console.log(`[${step}/${totalSteps}] Compiling Python bytecode...`);+ try {+ run('uv run python -O -m compileall -q -j 0 services core nodes routers models middleware main.py constants.py', serverDir, 120000);+ } catch (err) {+ console.log(` Warning: bytecode compilation failed (non-fatal): ${err.message}`);+ }+// WhatsApp RPC is now an npm dependency - binary downloaded via postinstallconsole.log('');console.log('Done!');diff --git a/server/nodejs/.gitignore b/server/nodejs/.gitignorenew file mode 100644index 00000000..3ad4c840--- /dev/null+++ b/server/nodejs/.gitignore@@ -0,0 +1,2 @@+dist/+package-lock.jsondiff --git a/server/nodejs/package.json b/server/nodejs/package.jsonindex 66caed79..2c2025fc 100644--- a/server/nodejs/package.json+++ b/server/nodejs/package.json@@ -3,9 +3,10 @@"version": "1.0.0","description": "Persistent Node.js server for JavaScript/TypeScript code execution","type": "module",- "main": "src/index.ts",+ "main": "dist/index.js","scripts": {- "start": "tsx src/index.ts",+ "build": "esbuild src/index.ts --bundle --platform=node --target=node22 --format=esm --packages=external --outfile=dist/index.js",+ "start": "node dist/index.js","dev": "tsx watch src/index.ts"},"dependencies": {@@ -14,6 +15,7 @@"devDependencies": {"@types/express": "^5.0.0","@types/node": "^22.0.0",+ "esbuild": "^0.24.0","tsx": "^4.21.0","typescript": "^5.7.0"},