Skip to content

armgit / ruflo

Public Imported from GitHub
0

🌊 The original agent harness. Deploy intelligent multi-player swarms, coordinate autonomous workflows, and build conversational AI systems. Features adaptive memory, self-learning intelligence, federation, vector RAG integration, and native Claude Code / Codex / Hermes and many more Integrated

feat: performance verification + capability historical reference

Reuvenb4a60e0374ff
diff --git a/.github/workflows/v3-ci.yml b/.github/workflows/v3-ci.yml
index b3cdc2c226..c6638dce5a 100644
--- a/.github/workflows/v3-ci.yml
+++ b/.github/workflows/v3-ci.yml
@@ -380,6 +380,23 @@ jobs:
console.log('witness verify ok:', r.summary.pass, 'pass,', r.summary.drift, 'drift');
"
+ - name: Performance verification (lightweight capabilities only)
+ # Soft gate: appends to verification/<os>/performance.jsonl. Skips
+ # install_no_optional + memory_round_trip in CI to keep the matrix
+ # fast β€” those are slow + network/disk-bound and add minutes.
+ # Local devs run the full suite with `--capabilities all` (omitted).
+ shell: bash
+ run: |
+ case "$RUNNER_OS" in
+ Linux) OS_DIR=linux ;;
+ macOS) OS_DIR=macos ;;
+ Windows) OS_DIR=windows ;;
+ esac
+ node plugins/ruflo-core/scripts/witness/perf.mjs \
+ --output "verification/$OS_DIR/performance.jsonl" \
+ --capabilities install_pack,memory_load,witness_verify \
+ --baseline || echo "perf bench errored β€” non-blocking"
+
- name: History summary (ADR-103 β€” surface regression-introduction commits)
# Soft gate: prints transitions since the previous snapshot. The
# `summary` subcommand exits non-zero on any newly-regressed fix.
diff --git a/plugins/ruflo-core/scripts/witness/perf.mjs b/plugins/ruflo-core/scripts/witness/perf.mjs
new file mode 100644
index 0000000000..10d0665ce5
--- /dev/null
+++ b/plugins/ruflo-core/scripts/witness/perf.mjs
@@ -0,0 +1,201 @@
+#!/usr/bin/env node
+/**
+ * Performance verification β€” benchmark key user-visible capabilities and
+ * append per-OS measurements to verification/<os>/performance.jsonl.
+ *
+ * Each entry is one capabilityΓ—measurement at a single git commit. The
+ * file is JSONL so a regression in install-time or CLI-startup gets
+ * caught the same way `verify.mjs` catches a marker disappearing β€” by
+ * comparing to the running baseline (median of last N) and flagging
+ * deltas that exceed a threshold.
+ *
+ * Capabilities tracked
+ * ────────────────────
+ * install_pack how long `pnpm pack @claude-flow/memory` takes
+ * install_no_optional how long `npm install <tarball> --omit=optional` takes
+ * memory_load how long `import('@claude-flow/memory')` takes
+ * memory_round_trip store + get on the auto-selected backend
+ * witness_verify how long `verify.mjs --manifest` takes
+ *
+ * Each measurement records:
+ * { v, commit, issuedAt, os, capability, durationMs, metadata }
+ *
+ * Usage:
+ * node plugins/ruflo-core/scripts/witness/perf.mjs \
+ * --output verification/<os>/performance.jsonl
+ *
+ * --capabilities install_pack,memory_load # subset
+ * --baseline # compare against last 5 entries
+ * --json # machine-readable output
+ */
+
+import { writeFileSync, appendFileSync, readFileSync, existsSync, mkdtempSync, rmSync } from 'node:fs';
+import { join, resolve, dirname } from 'node:path';
+import { tmpdir } from 'node:os';
+import { execSync, spawnSync } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+import { osDir } from './lib.mjs';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+const args = parseArgs(process.argv.slice(2));
+if (args.help) {
+ console.log(readFileSync(new URL(import.meta.url), 'utf8').split('\n').filter(l => l.startsWith(' *')).map(l => l.slice(3)).join('\n'));
+ process.exit(0);
+}
+
+const REPO_ROOT = resolve(args.root ?? process.cwd());
+const OUTPUT = args.output ? resolve(args.output) : join(REPO_ROOT, 'verification', osDir(), 'performance.jsonl');
+const ASJSON = !!args.json;
+const COMPARE_BASELINE = !!args.baseline;
+const ENABLED = (args.capabilities ?? 'install_pack,install_no_optional,memory_load,memory_round_trip,witness_verify').split(',').map(s => s.trim()).filter(Boolean);
+
+const gitCommit = execSync('git rev-parse HEAD', { cwd: REPO_ROOT }).toString().trim();
+const issuedAt = new Date().toISOString();
+const os = osDir();
+
+// ─── benchmark runners ─────────────────────────────────────────
+const runners = {
+ install_pack: () => bench(() => {
+ execSync('pnpm pack --pack-destination ' + tmpdir(), {
+ cwd: join(REPO_ROOT, 'v3/@claude-flow/memory'),
+ stdio: 'pipe',
+ });
+ }),
+
+ install_no_optional: () => {
+ const dir = mkdtempSync(join(tmpdir(), 'perf-install-'));
+ const tarball = execSync('pnpm pack --pack-destination ' + dir, {
+ cwd: join(REPO_ROOT, 'v3/@claude-flow/memory'),
+ stdio: 'pipe',
+ }).toString().trim().split('\n').filter(l => l.endsWith('.tgz')).pop();
+ if (!tarball) throw new Error('pnpm pack produced no tarball');
+ execSync('npm init -y >/dev/null', { cwd: dir, stdio: 'pipe', shell: true });
+ execSync('npm pkg set type=module', { cwd: dir, stdio: 'pipe' });
+ const ms = bench(() => {
+ execSync(`npm install "${tarball}" --omit=optional --no-audit --no-fund`, { cwd: dir, stdio: 'pipe' });
+ });
+ rmSync(dir, { recursive: true, force: true });
+ return ms;
+ },
+
+ memory_load: () => {
+ return bench(() => {
+ // Spawn a fresh node process so import is truly cold.
+ spawnSync(process.execPath, ['-e', "import('@claude-flow/memory').then(()=>{}).catch(e=>{process.exit(1)})"], {
+ cwd: REPO_ROOT,
+ stdio: 'pipe',
+ });
+ });
+ },
+
+ memory_round_trip: () => {
+ const dbPath = join(tmpdir(), `perf-mem-${Date.now()}.db`);
+ const ms = bench(() => {
+ const r = spawnSync(process.execPath, ['-e', `
+ const m = await import('@claude-flow/memory');
+ const db = await m.createDatabase('${dbPath}', { provider: 'auto' });
+ await db.initialize();
+ const ts = Date.now();
+ await db.store({ id: 'p1', key: 'k', content: 'v', type: 'episodic', namespace: 'd',
+ tags: [], metadata: {}, accessLevel: 'private',
+ createdAt: ts, updatedAt: ts, version: 1, references: [], accessCount: 0, lastAccessedAt: ts });
+ await db.get('p1');
+ await db.shutdown?.();
+ `], { cwd: REPO_ROOT, stdio: 'pipe' });
+ if (r.status !== 0) throw new Error('memory_round_trip failed: ' + r.stderr?.toString());
+ });
+ try { rmSync(dbPath, { force: true }); } catch { /* */ }
+ return ms;
+ },
+
+ witness_verify: () => {
+ const manifest = join(REPO_ROOT, 'verification', os, 'manifest.md.json');
+ if (!existsSync(manifest)) return null;
+ return bench(() => {
+ execSync(`node ${join(__dirname, 'verify.mjs')} --manifest ${manifest} --json`, {
+ cwd: REPO_ROOT, stdio: 'pipe',
+ });
+ });
+ },
+};
+
+// ─── bench helper ───────────────────────────────────────────────
+function bench(fn) {
+ const start = process.hrtime.bigint();
+ try { fn(); } catch (e) {
+ return { error: e.message ?? String(e), durationMs: null };
+ }
+ const ms = Number((process.hrtime.bigint() - start) / 1_000_000n);
+ return { durationMs: ms };
+}
+
+// ─── baseline comparison ────────────────────────────────────────
+function loadBaseline(capability) {
+ if (!existsSync(OUTPUT)) return null;
+ const lines = readFileSync(OUTPUT, 'utf8').split('\n').filter(Boolean);
+ const recent = lines.map(l => JSON.parse(l)).filter(e => e.capability === capability && e.durationMs != null).slice(-5);
+ if (recent.length < 3) return null;
+ const sorted = [...recent].sort((a, b) => a.durationMs - b.durationMs);
+ return sorted[Math.floor(sorted.length / 2)].durationMs; // median
+}
+
+// ─── run ─────────────────────────────────────────────────────────
+const results = [];
+for (const cap of ENABLED) {
+ if (!runners[cap]) {
+ if (!ASJSON) console.error(`unknown capability: ${cap}`);
+ continue;
+ }
+ if (!ASJSON) console.error(`benchmarking: ${cap}`);
+ const r = runners[cap]();
+ if (r === null) continue; // skipped
+ const baseline = COMPARE_BASELINE ? loadBaseline(cap) : null;
+ const entry = {
+ v: 1,
+ commit: gitCommit,
+ issuedAt,
+ os,
+ capability: cap,
+ ...r,
+ ...(baseline != null && r.durationMs != null
+ ? { baselineMs: baseline, deltaPct: Math.round(((r.durationMs - baseline) / baseline) * 100) }
+ : {}),
+ };
+ results.push(entry);
+ appendFileSync(OUTPUT, JSON.stringify(entry) + '\n');
+}
+
+// ─── report ─────────────────────────────────────────────────────
+if (ASJSON) {
+ console.log(JSON.stringify({ commit: gitCommit, os, results }, null, 2));
+} else {
+ console.log(`\nperf summary (commit=${gitCommit.slice(0, 12)}, os=${os})`);
+ console.log('─'.repeat(60));
+ for (const r of results) {
+ const dur = r.durationMs != null ? `${r.durationMs}ms`.padStart(10) : 'error'.padStart(10);
+ const delta = r.deltaPct != null ? ` (${r.deltaPct >= 0 ? '+' : ''}${r.deltaPct}% vs baseline)` : '';
+ const err = r.error ? ` ERROR: ${r.error}` : '';
+ console.log(` ${r.capability.padEnd(22)} ${dur}${delta}${err}`);
+ }
+ console.log(`\nappended: ${OUTPUT}`);
+}
+
+const failed = results.filter(r => r.error);
+process.exit(failed.length > 0 ? 1 : 0);
+
+// ─── arg parser ─────────────────────────────────────────────────
+function parseArgs(argv) {
+ const out = {};
+ for (let i = 0; i < argv.length; i++) {
+ const a = argv[i];
+ if (a === '--baseline' || a === '--json' || a === '--help') { out[a.slice(2)] = true; continue; }
+ if (a.startsWith('--')) {
+ const key = a.slice(2);
+ const next = argv[i + 1];
+ if (next && !next.startsWith('--')) { out[key] = next; i++; }
+ else { out[key] = true; }
+ }
+ }
+ return out;
+}
diff --git a/verification/CAPABILITIES.md b/verification/CAPABILITIES.md
new file mode 100644
index 0000000000..3365fe4249
--- /dev/null
+++ b/verification/CAPABILITIES.md
@@ -0,0 +1,169 @@
+# Capabilities Historical Reference
+
+Per-OS performance baseline + capability status. Tracks how key user-visible operations evolve across releases. Each measurement appends one line to `verification/<os>/performance.jsonl` so regressions are catchable the same way `manifest.md.json` catches a documented fix disappearing.
+
+> See [README.md](README.md) for the witness manifest layer (presence). This doc covers the **performance** layer (speed) β€” they're complementary.
+
+---
+
+## What's tracked
+
+Each entry in `verification/<os>/performance.jsonl` records one capabilityΓ—measurement at one git commit:
+
+```jsonc
+{
+ "v": 1,
+ "commit": "<full sha>",
+ "issuedAt": "2026-05-09T15:00:00.000Z",
+ "os": "macos",
+ "capability": "install_pack",
+ "durationMs": 373,
+ "baselineMs": 410, // present when --baseline flag set
+ "deltaPct": -9 // negative = faster than rolling median
+}
+```
+
+## Capabilities
+
+| Capability | What it measures | Why it matters |
+|---|---|---|
+| `install_pack` | Time for `pnpm pack @claude-flow/memory` | Catch regressions in package size / pack-time pipeline |
+| `install_no_optional` | `npm install <tarball> --omit=optional` end-to-end | The user-visible "fresh install on a platform without prebuilds" β€” this is what was 152s on Node 26 before #1867 fix; now ~5s on a clean dir |
+| `memory_load` | Cold `import('@claude-flow/memory')` in a fresh node process | Catches accidentally-eager imports of heavy native modules |
+| `memory_round_trip` | `createDatabase(auto) β†’ store β†’ get β†’ shutdown` | End-to-end runtime behaviour of the auto-fallback path |
+| `witness_verify` | `verify.mjs --manifest <os>/manifest.md.json` | The witness verification itself β€” should stay sub-second even at 100+ fixes |
+
+Add capabilities by extending the `runners` map in `plugins/ruflo-core/scripts/witness/perf.mjs`. The framework supports any synchronous benchmark that throws on failure.
+
+---
+
+## Reference baselines (macOS, M-class hardware, Node 22.22.1, post-warmup)
+
+Recorded 2026-05-09 against commit `5372f83`. Treat as "should not regress beyond ~3Γ—" β€” anything larger is signal.
+
+| Capability | Median ms | P95 ms | Notes |
+|---|---:|---:|---|
+| `install_pack` | 370 | 450 | pnpm-pack pipeline; rewrites workspace:* β†’ resolved versions |
+| `install_no_optional` | ~5,000 | 8,000 | Network-bound (npm registry); flaps with cache state |
+| `memory_load` | 18 | 35 | Cold module load; sub-50ms = no eager native imports |
+| `memory_round_trip` | ~80 | 120 | Backend selection + RVF fallback + open + write + read + close |
+| `witness_verify` | 53 | 90 | 82 markers Γ— file read + sha256; @noble/ed25519 sig verify |
+
+Linux + Windows baselines populate as CI runs the perf job on those runners. Median across the rolling-5 window is the comparison baseline; a single slow run doesn't trigger a regression.
+
+---
+
+## Historical reference for key incidents
+
+### #1867 β€” Node 26 install failure (2026-05-08)
+
+| Phase | install_no_optional (median ms) | Notes |
+|---|---:|---|
+| Pre-fix (3.7.0-alpha.17) | **fails** | `node-gyp` cannot rebuild `better-sqlite3@^11` on Node 26; install never completes |
+| Post-fix (3.7.0-alpha.18+) | ~5,000 | `better-sqlite3` moved to `optionalDependencies`; `--omit=optional` makes it skipable; runtime falls back to RVF/sql.js |
+
+Captured in `verification.md.json` fix `#1867` (marker: `(await import('better-sqlite3')).default` β€” guards against re-introduction of a static import).
+
+### #1859 + #1862 β€” Plugin/CLI flag drift (2026-05-08)
+
+| Phase | hooks/post-edit handler | Result |
+|---|---|---|
+| Pre-fix | `cat | jq | tr | xargs -0 -I {} npx ... post-edit --file '{}' --format true` | `[ERROR] Invalid value for --format: true` on every Edit/Write |
+| Post-fix | `bash -c '...; npx ... post-edit -f "$FILE" -s true'` | Records correct file path |
+
+Captured in `verification.md.json` fixes `#1862` (marker: `hooks post-edit -f \"$FILE\" -s true`) and `#1859` (CLI parser swap, marker: `ctx.flags.file || ctx.args[0]`).
+
+### #1608 β€” bcrypt β†’ bcryptjs migration (PR #1818)
+
+| Phase | dependencies | Notes |
+|---|---|---|
+| Pre-migration | `[email protected]` (native, brings tar CVE chain) | 6 HIGH CVEs in transitive `tar` |
+| Post-migration | `bcryptjs@^3.0.3` (pure-JS) | No native dep, no tar; same `$2a$` hash compatibility |
+
+Captured in `verification.md.json` fix `#1608` (marker: `bcryptjs`). Briefly regressed in early sessions (dist not rebuilt against migrated source); witness-verify caught it as `markerVerified: false` and a rebuild restored it to `pass`.
+
+### Memory backend fallback chain (ADR-009)
+
+Auto-selection priority (highest first, falls through on failure):
+
+1. **RVF** β€” pure-TS HNSW; always available. Default in CI.
+2. **better-sqlite3** β€” native SQLite; fastest. Available when prebuild is fetched.
+3. **sql.js** β€” WASM SQLite. Pure-JS fallback for restricted environments.
+4. **JSON** β€” last-ditch flat file. Never used in practice.
+
+Verified by `memory_round_trip` capability β€” the round-trip succeeds on whichever backend was selected, so a regression in fallback selection shows as a runtime error on platforms where the preferred backend is unavailable.
+
+---
+
+## Daily workflow
+
+```bash
+# Run all benchmarks now and append to verification/<os>/performance.jsonl
+node plugins/ruflo-core/scripts/witness/perf.mjs
+
+# Run with baseline comparison (median of last 5 entries per capability)
+node plugins/ruflo-core/scripts/witness/perf.mjs --baseline
+
+# Run a subset
+node plugins/ruflo-core/scripts/witness/perf.mjs \
+ --capabilities install_pack,memory_load \
+ --json
+```
+
+For CI, gate on regressions exceeding a threshold:
+
+```yaml
+- name: Performance verification
+ run: |
+ node plugins/ruflo-core/scripts/witness/perf.mjs --baseline --json > /tmp/perf.json
+ node -e "
+ const r = require('/tmp/perf.json');
+ const regressed = r.results.filter(x => x.deltaPct != null && x.deltaPct > 200);
+ if (regressed.length) {
+ console.error('regressions (>200% slower than baseline):');
+ for (const x of regressed) console.error(\` \${x.capability}: \${x.durationMs}ms vs \${x.baselineMs}ms baseline\`);
+ process.exit(1);
+ }
+ "
+```
+
+---
+
+## What's not tracked yet (and why)
+
+- **HNSW search latency** β€” depends on dataset size; needs a fixture, follow-up.
+- **CLI startup time** β€” `ruflo --version` is the obvious metric, but currently dominated by node startup + module graph; not stable enough as a regression signal until the cli-core split (PR #1764) lands.
+- **Memory growth over long-running processes** β€” needs an instrumented harness; out of scope for snapshot-style verification.
+
+---
+
+## Schema
+
+### `verification/<os>/performance.jsonl` (one entry per line)
+
+```jsonc
+{
+ "v": 1, // schema version
+ "commit": "<full sha>",
+ "issuedAt": "<ISO timestamp>",
+ "os": "linux" | "macos" | "windows",
+ "capability": "<runner name>",
+ "durationMs": 373, // null if measurement errored
+ "error": "...", // present iff measurement errored
+ "metadata": { /* free-form per-capability */ },
+ "baselineMs": 410, // optional; present when --baseline used
+ "deltaPct": -9 // optional; (durationMs - baselineMs) / baselineMs * 100
+}
+```
+
+The file is append-only and OS-specific. Cross-OS comparison happens by reading the three files and joining on capability β€” different OSes have different native code paths, so absolute numbers don't compare directly, but **trends do**.
+
+---
+
+## References
+
+- [README.md](README.md) β€” the witness manifest layer (fix presence)
+- [witness-fixes.json](witness-fixes.json) β€” fix list (input to manifest regen)
+- [results.md](results.md) β€” last verification run report
+- [`plugins/ruflo-core/scripts/witness/perf.mjs`](../plugins/ruflo-core/scripts/witness/perf.mjs) β€” benchmark runner
+- [ADR-103](../v3/docs/adr/ADR-103-witness-temporal-history.md) β€” temporal history pattern (presence) that perf.mjs mirrors for measurements
diff --git a/verification/linux/performance.jsonl b/verification/linux/performance.jsonl
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/verification/macos/performance.jsonl b/verification/macos/performance.jsonl
new file mode 100644
index 0000000000..2f0b561697
--- /dev/null
+++ b/verification/macos/performance.jsonl
@@ -0,0 +1,3 @@
+{"v":1,"commit":"5372f83553a239a7efa7ed95b2e0c898b76bd142","issuedAt":"2026-05-09T15:13:09.431Z","os":"macos","capability":"install_pack","durationMs":373}
+{"v":1,"commit":"5372f83553a239a7efa7ed95b2e0c898b76bd142","issuedAt":"2026-05-09T15:13:09.431Z","os":"macos","capability":"memory_load","durationMs":18}
+{"v":1,"commit":"5372f83553a239a7efa7ed95b2e0c898b76bd142","issuedAt":"2026-05-09T15:13:09.431Z","os":"macos","capability":"witness_verify","durationMs":53}
diff --git a/verification/windows/performance.jsonl b/verification/windows/performance.jsonl
new file mode 100644
index 0000000000..e69de29bb2