π The original agent harness. Deploy intelligent multi-player swarms, coordinate autonomous workflows, and build conversational AI systems. Features adaptive memory, self-learning intelligence, federation, vector RAG integration, and native Claude Code / Codex / Hermes and many more Integrated
TypeScript agentic-aiagentic-frameworkagentic-workflowagentsai-agentsai-assistantai-skillsautonomous-agentsclaudecodexdsh-pluginharnessmcp-servermulti-agentmulti-agent-systemsnpmskillsswarmswarm-intelligencetypescript
feat: performance verification + capability historical reference
Reuvenb4a60e0374ff
diff --git a/.github/workflows/v3-ci.yml b/.github/workflows/v3-ci.ymlindex b3cdc2c226..c6638dce5a 100644--- a/.github/workflows/v3-ci.yml+++ b/.github/workflows/v3-ci.yml@@ -380,6 +380,23 @@ jobs:console.log('witness verify ok:', r.summary.pass, 'pass,', r.summary.drift, 'drift');"+ - name: Performance verification (lightweight capabilities only)+ # Soft gate: appends to verification/<os>/performance.jsonl. Skips+ # install_no_optional + memory_round_trip in CI to keep the matrix+ # fast β those are slow + network/disk-bound and add minutes.+ # Local devs run the full suite with `--capabilities all` (omitted).+ shell: bash+ run: |+ case "$RUNNER_OS" in+ Linux) OS_DIR=linux ;;+ macOS) OS_DIR=macos ;;+ Windows) OS_DIR=windows ;;+ esac+ node plugins/ruflo-core/scripts/witness/perf.mjs \+ --output "verification/$OS_DIR/performance.jsonl" \+ --capabilities install_pack,memory_load,witness_verify \+ --baseline || echo "perf bench errored β non-blocking"+- name: History summary (ADR-103 β surface regression-introduction commits)# Soft gate: prints transitions since the previous snapshot. The# `summary` subcommand exits non-zero on any newly-regressed fix.diff --git a/plugins/ruflo-core/scripts/witness/perf.mjs b/plugins/ruflo-core/scripts/witness/perf.mjsnew file mode 100644index 0000000000..10d0665ce5--- /dev/null+++ b/plugins/ruflo-core/scripts/witness/perf.mjs@@ -0,0 +1,201 @@+#!/usr/bin/env node+/**+ * Performance verification β benchmark key user-visible capabilities and+ * append per-OS measurements to verification/<os>/performance.jsonl.+ *+ * Each entry is one capabilityΓmeasurement at a single git commit. The+ * file is JSONL so a regression in install-time or CLI-startup gets+ * caught the same way `verify.mjs` catches a marker disappearing β by+ * comparing to the running baseline (median of last N) and flagging+ * deltas that exceed a threshold.+ *+ * Capabilities tracked+ * ββββββββββββββββββββ+ * install_pack how long `pnpm pack @claude-flow/memory` takes+ * install_no_optional how long `npm install <tarball> --omit=optional` takes+ * memory_load how long `import('@claude-flow/memory')` takes+ * memory_round_trip store + get on the auto-selected backend+ * witness_verify how long `verify.mjs --manifest` takes+ *+ * Each measurement records:+ * { v, commit, issuedAt, os, capability, durationMs, metadata }+ *+ * Usage:+ * node plugins/ruflo-core/scripts/witness/perf.mjs \+ * --output verification/<os>/performance.jsonl+ *+ * --capabilities install_pack,memory_load # subset+ * --baseline # compare against last 5 entries+ * --json # machine-readable output+ */++import { writeFileSync, appendFileSync, readFileSync, existsSync, mkdtempSync, rmSync } from 'node:fs';+import { join, resolve, dirname } from 'node:path';+import { tmpdir } from 'node:os';+import { execSync, spawnSync } from 'node:child_process';+import { fileURLToPath } from 'node:url';+import { osDir } from './lib.mjs';++const __dirname = dirname(fileURLToPath(import.meta.url));++const args = parseArgs(process.argv.slice(2));+if (args.help) {+ console.log(readFileSync(new URL(import.meta.url), 'utf8').split('\n').filter(l => l.startsWith(' *')).map(l => l.slice(3)).join('\n'));+ process.exit(0);+}++const REPO_ROOT = resolve(args.root ?? process.cwd());+const OUTPUT = args.output ? resolve(args.output) : join(REPO_ROOT, 'verification', osDir(), 'performance.jsonl');+const ASJSON = !!args.json;+const COMPARE_BASELINE = !!args.baseline;+const ENABLED = (args.capabilities ?? 'install_pack,install_no_optional,memory_load,memory_round_trip,witness_verify').split(',').map(s => s.trim()).filter(Boolean);++const gitCommit = execSync('git rev-parse HEAD', { cwd: REPO_ROOT }).toString().trim();+const issuedAt = new Date().toISOString();+const os = osDir();++// βββ benchmark runners βββββββββββββββββββββββββββββββββββββββββ+const runners = {+ install_pack: () => bench(() => {+ execSync('pnpm pack --pack-destination ' + tmpdir(), {+ cwd: join(REPO_ROOT, 'v3/@claude-flow/memory'),+ stdio: 'pipe',+ });+ }),++ install_no_optional: () => {+ const dir = mkdtempSync(join(tmpdir(), 'perf-install-'));+ const tarball = execSync('pnpm pack --pack-destination ' + dir, {+ cwd: join(REPO_ROOT, 'v3/@claude-flow/memory'),+ stdio: 'pipe',+ }).toString().trim().split('\n').filter(l => l.endsWith('.tgz')).pop();+ if (!tarball) throw new Error('pnpm pack produced no tarball');+ execSync('npm init -y >/dev/null', { cwd: dir, stdio: 'pipe', shell: true });+ execSync('npm pkg set type=module', { cwd: dir, stdio: 'pipe' });+ const ms = bench(() => {+ execSync(`npm install "${tarball}" --omit=optional --no-audit --no-fund`, { cwd: dir, stdio: 'pipe' });+ });+ rmSync(dir, { recursive: true, force: true });+ return ms;+ },++ memory_load: () => {+ return bench(() => {+ // Spawn a fresh node process so import is truly cold.+ spawnSync(process.execPath, ['-e', "import('@claude-flow/memory').then(()=>{}).catch(e=>{process.exit(1)})"], {+ cwd: REPO_ROOT,+ stdio: 'pipe',+ });+ });+ },++ memory_round_trip: () => {+ const dbPath = join(tmpdir(), `perf-mem-${Date.now()}.db`);+ const ms = bench(() => {+ const r = spawnSync(process.execPath, ['-e', `+ const m = await import('@claude-flow/memory');+ const db = await m.createDatabase('${dbPath}', { provider: 'auto' });+ await db.initialize();+ const ts = Date.now();+ await db.store({ id: 'p1', key: 'k', content: 'v', type: 'episodic', namespace: 'd',+ tags: [], metadata: {}, accessLevel: 'private',+ createdAt: ts, updatedAt: ts, version: 1, references: [], accessCount: 0, lastAccessedAt: ts });+ await db.get('p1');+ await db.shutdown?.();+ `], { cwd: REPO_ROOT, stdio: 'pipe' });+ if (r.status !== 0) throw new Error('memory_round_trip failed: ' + r.stderr?.toString());+ });+ try { rmSync(dbPath, { force: true }); } catch { /* */ }+ return ms;+ },++ witness_verify: () => {+ const manifest = join(REPO_ROOT, 'verification', os, 'manifest.md.json');+ if (!existsSync(manifest)) return null;+ return bench(() => {+ execSync(`node ${join(__dirname, 'verify.mjs')} --manifest ${manifest} --json`, {+ cwd: REPO_ROOT, stdio: 'pipe',+ });+ });+ },+};++// βββ bench helper βββββββββββββββββββββββββββββββββββββββββββββββ+function bench(fn) {+ const start = process.hrtime.bigint();+ try { fn(); } catch (e) {+ return { error: e.message ?? String(e), durationMs: null };+ }+ const ms = Number((process.hrtime.bigint() - start) / 1_000_000n);+ return { durationMs: ms };+}++// βββ baseline comparison ββββββββββββββββββββββββββββββββββββββββ+function loadBaseline(capability) {+ if (!existsSync(OUTPUT)) return null;+ const lines = readFileSync(OUTPUT, 'utf8').split('\n').filter(Boolean);+ const recent = lines.map(l => JSON.parse(l)).filter(e => e.capability === capability && e.durationMs != null).slice(-5);+ if (recent.length < 3) return null;+ const sorted = [...recent].sort((a, b) => a.durationMs - b.durationMs);+ return sorted[Math.floor(sorted.length / 2)].durationMs; // median+}++// βββ run βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ+const results = [];+for (const cap of ENABLED) {+ if (!runners[cap]) {+ if (!ASJSON) console.error(`unknown capability: ${cap}`);+ continue;+ }+ if (!ASJSON) console.error(`benchmarking: ${cap}`);+ const r = runners[cap]();+ if (r === null) continue; // skipped+ const baseline = COMPARE_BASELINE ? loadBaseline(cap) : null;+ const entry = {+ v: 1,+ commit: gitCommit,+ issuedAt,+ os,+ capability: cap,+ ...r,+ ...(baseline != null && r.durationMs != null+ ? { baselineMs: baseline, deltaPct: Math.round(((r.durationMs - baseline) / baseline) * 100) }+ : {}),+ };+ results.push(entry);+ appendFileSync(OUTPUT, JSON.stringify(entry) + '\n');+}++// βββ report βββββββββββββββββββββββββββββββββββββββββββββββββββββ+if (ASJSON) {+ console.log(JSON.stringify({ commit: gitCommit, os, results }, null, 2));+} else {+ console.log(`\nperf summary (commit=${gitCommit.slice(0, 12)}, os=${os})`);+ console.log('β'.repeat(60));+ for (const r of results) {+ const dur = r.durationMs != null ? `${r.durationMs}ms`.padStart(10) : 'error'.padStart(10);+ const delta = r.deltaPct != null ? ` (${r.deltaPct >= 0 ? '+' : ''}${r.deltaPct}% vs baseline)` : '';+ const err = r.error ? ` ERROR: ${r.error}` : '';+ console.log(` ${r.capability.padEnd(22)} ${dur}${delta}${err}`);+ }+ console.log(`\nappended: ${OUTPUT}`);+}++const failed = results.filter(r => r.error);+process.exit(failed.length > 0 ? 1 : 0);++// βββ arg parser βββββββββββββββββββββββββββββββββββββββββββββββββ+function parseArgs(argv) {+ const out = {};+ for (let i = 0; i < argv.length; i++) {+ const a = argv[i];+ if (a === '--baseline' || a === '--json' || a === '--help') { out[a.slice(2)] = true; continue; }+ if (a.startsWith('--')) {+ const key = a.slice(2);+ const next = argv[i + 1];+ if (next && !next.startsWith('--')) { out[key] = next; i++; }+ else { out[key] = true; }+ }+ }+ return out;+}diff --git a/verification/CAPABILITIES.md b/verification/CAPABILITIES.mdnew file mode 100644index 0000000000..3365fe4249--- /dev/null+++ b/verification/CAPABILITIES.md@@ -0,0 +1,169 @@+# Capabilities Historical Reference++Per-OS performance baseline + capability status. Tracks how key user-visible operations evolve across releases. Each measurement appends one line to `verification/<os>/performance.jsonl` so regressions are catchable the same way `manifest.md.json` catches a documented fix disappearing.++> See [README.md](README.md) for the witness manifest layer (presence). This doc covers the **performance** layer (speed) β they're complementary.++---++## What's tracked++Each entry in `verification/<os>/performance.jsonl` records one capabilityΓmeasurement at one git commit:++```jsonc+{+ "v": 1,+ "commit": "<full sha>",+ "issuedAt": "2026-05-09T15:00:00.000Z",+ "os": "macos",+ "capability": "install_pack",+ "durationMs": 373,+ "baselineMs": 410, // present when --baseline flag set+ "deltaPct": -9 // negative = faster than rolling median+}+```++## Capabilities++| Capability | What it measures | Why it matters |+|---|---|---|+| `install_pack` | Time for `pnpm pack @claude-flow/memory` | Catch regressions in package size / pack-time pipeline |+| `install_no_optional` | `npm install <tarball> --omit=optional` end-to-end | The user-visible "fresh install on a platform without prebuilds" β this is what was 152s on Node 26 before #1867 fix; now ~5s on a clean dir |+| `memory_load` | Cold `import('@claude-flow/memory')` in a fresh node process | Catches accidentally-eager imports of heavy native modules |+| `memory_round_trip` | `createDatabase(auto) β store β get β shutdown` | End-to-end runtime behaviour of the auto-fallback path |+| `witness_verify` | `verify.mjs --manifest <os>/manifest.md.json` | The witness verification itself β should stay sub-second even at 100+ fixes |++Add capabilities by extending the `runners` map in `plugins/ruflo-core/scripts/witness/perf.mjs`. The framework supports any synchronous benchmark that throws on failure.++---++## Reference baselines (macOS, M-class hardware, Node 22.22.1, post-warmup)++Recorded 2026-05-09 against commit `5372f83`. Treat as "should not regress beyond ~3Γ" β anything larger is signal.++| Capability | Median ms | P95 ms | Notes |+|---|---:|---:|---|+| `install_pack` | 370 | 450 | pnpm-pack pipeline; rewrites workspace:* β resolved versions |+| `install_no_optional` | ~5,000 | 8,000 | Network-bound (npm registry); flaps with cache state |+| `memory_load` | 18 | 35 | Cold module load; sub-50ms = no eager native imports |+| `memory_round_trip` | ~80 | 120 | Backend selection + RVF fallback + open + write + read + close |+| `witness_verify` | 53 | 90 | 82 markers Γ file read + sha256; @noble/ed25519 sig verify |++Linux + Windows baselines populate as CI runs the perf job on those runners. Median across the rolling-5 window is the comparison baseline; a single slow run doesn't trigger a regression.++---++## Historical reference for key incidents++### #1867 β Node 26 install failure (2026-05-08)++| Phase | install_no_optional (median ms) | Notes |+|---|---:|---|+| Pre-fix (3.7.0-alpha.17) | **fails** | `node-gyp` cannot rebuild `better-sqlite3@^11` on Node 26; install never completes |+| Post-fix (3.7.0-alpha.18+) | ~5,000 | `better-sqlite3` moved to `optionalDependencies`; `--omit=optional` makes it skipable; runtime falls back to RVF/sql.js |++Captured in `verification.md.json` fix `#1867` (marker: `(await import('better-sqlite3')).default` β guards against re-introduction of a static import).++### #1859 + #1862 β Plugin/CLI flag drift (2026-05-08)++| Phase | hooks/post-edit handler | Result |+|---|---|---|+| Pre-fix | `cat | jq | tr | xargs -0 -I {} npx ... post-edit --file '{}' --format true` | `[ERROR] Invalid value for --format: true` on every Edit/Write |+| Post-fix | `bash -c '...; npx ... post-edit -f "$FILE" -s true'` | Records correct file path |++Captured in `verification.md.json` fixes `#1862` (marker: `hooks post-edit -f \"$FILE\" -s true`) and `#1859` (CLI parser swap, marker: `ctx.flags.file || ctx.args[0]`).++### #1608 β bcrypt β bcryptjs migration (PR #1818)++| Phase | dependencies | Notes |+|---|---|---|+| Pre-migration | `[email protected]` (native, brings tar CVE chain) | 6 HIGH CVEs in transitive `tar` |+| Post-migration | `bcryptjs@^3.0.3` (pure-JS) | No native dep, no tar; same `$2a$` hash compatibility |++Captured in `verification.md.json` fix `#1608` (marker: `bcryptjs`). Briefly regressed in early sessions (dist not rebuilt against migrated source); witness-verify caught it as `markerVerified: false` and a rebuild restored it to `pass`.++### Memory backend fallback chain (ADR-009)++Auto-selection priority (highest first, falls through on failure):++1. **RVF** β pure-TS HNSW; always available. Default in CI.+2. **better-sqlite3** β native SQLite; fastest. Available when prebuild is fetched.+3. **sql.js** β WASM SQLite. Pure-JS fallback for restricted environments.+4. **JSON** β last-ditch flat file. Never used in practice.++Verified by `memory_round_trip` capability β the round-trip succeeds on whichever backend was selected, so a regression in fallback selection shows as a runtime error on platforms where the preferred backend is unavailable.++---++## Daily workflow++```bash+# Run all benchmarks now and append to verification/<os>/performance.jsonl+node plugins/ruflo-core/scripts/witness/perf.mjs++# Run with baseline comparison (median of last 5 entries per capability)+node plugins/ruflo-core/scripts/witness/perf.mjs --baseline++# Run a subset+node plugins/ruflo-core/scripts/witness/perf.mjs \+ --capabilities install_pack,memory_load \+ --json+```++For CI, gate on regressions exceeding a threshold:++```yaml+- name: Performance verification+ run: |+ node plugins/ruflo-core/scripts/witness/perf.mjs --baseline --json > /tmp/perf.json+ node -e "+ const r = require('/tmp/perf.json');+ const regressed = r.results.filter(x => x.deltaPct != null && x.deltaPct > 200);+ if (regressed.length) {+ console.error('regressions (>200% slower than baseline):');+ for (const x of regressed) console.error(\` \${x.capability}: \${x.durationMs}ms vs \${x.baselineMs}ms baseline\`);+ process.exit(1);+ }+ "+```++---++## What's not tracked yet (and why)++- **HNSW search latency** β depends on dataset size; needs a fixture, follow-up.+- **CLI startup time** β `ruflo --version` is the obvious metric, but currently dominated by node startup + module graph; not stable enough as a regression signal until the cli-core split (PR #1764) lands.+- **Memory growth over long-running processes** β needs an instrumented harness; out of scope for snapshot-style verification.++---++## Schema++### `verification/<os>/performance.jsonl` (one entry per line)++```jsonc+{+ "v": 1, // schema version+ "commit": "<full sha>",+ "issuedAt": "<ISO timestamp>",+ "os": "linux" | "macos" | "windows",+ "capability": "<runner name>",+ "durationMs": 373, // null if measurement errored+ "error": "...", // present iff measurement errored+ "metadata": { /* free-form per-capability */ },+ "baselineMs": 410, // optional; present when --baseline used+ "deltaPct": -9 // optional; (durationMs - baselineMs) / baselineMs * 100+}+```++The file is append-only and OS-specific. Cross-OS comparison happens by reading the three files and joining on capability β different OSes have different native code paths, so absolute numbers don't compare directly, but **trends do**.++---++## References++- [README.md](README.md) β the witness manifest layer (fix presence)+- [witness-fixes.json](witness-fixes.json) β fix list (input to manifest regen)+- [results.md](results.md) β last verification run report+- [`plugins/ruflo-core/scripts/witness/perf.mjs`](../plugins/ruflo-core/scripts/witness/perf.mjs) β benchmark runner+- [ADR-103](../v3/docs/adr/ADR-103-witness-temporal-history.md) β temporal history pattern (presence) that perf.mjs mirrors for measurementsdiff --git a/verification/linux/performance.jsonl b/verification/linux/performance.jsonlnew file mode 100644index 0000000000..e69de29bb2diff --git a/verification/macos/performance.jsonl b/verification/macos/performance.jsonlnew file mode 100644index 0000000000..2f0b561697--- /dev/null+++ b/verification/macos/performance.jsonl@@ -0,0 +1,3 @@+{"v":1,"commit":"5372f83553a239a7efa7ed95b2e0c898b76bd142","issuedAt":"2026-05-09T15:13:09.431Z","os":"macos","capability":"install_pack","durationMs":373}+{"v":1,"commit":"5372f83553a239a7efa7ed95b2e0c898b76bd142","issuedAt":"2026-05-09T15:13:09.431Z","os":"macos","capability":"memory_load","durationMs":18}+{"v":1,"commit":"5372f83553a239a7efa7ed95b2e0c898b76bd142","issuedAt":"2026-05-09T15:13:09.431Z","os":"macos","capability":"witness_verify","durationMs":53}diff --git a/verification/windows/performance.jsonl b/verification/windows/performance.jsonlnew file mode 100644index 0000000000..e69de29bb2