ruvnet.
Python
docs(ruvnet): record rejected 2026-09-25 experiment
rUvb3353b21014f
diff --git a/docs/self-improvement/2026-09-25-benchmarks.json b/docs/self-improvement/2026-09-25-benchmarks.jsonnew file mode 100644index 0000000..d5dc77e--- /dev/null+++ b/docs/self-improvement/2026-09-25-benchmarks.json@@ -0,0 +1,458 @@+{+ "schemaVersion": 1,+ "seed": 1729,+ "seedUsage": "metadata only; fixed alternating order and no RNG",+ "pairs": 15,+ "hotCallsPerArmPerPair": 10000,+ "coldStartsPerArmPerPair": 20,+ "query": "memory quantize graph metadata learning router",+ "arms": {+ "baseline": [+ {+ "pair": 0,+ "hot": {+ "p50Ms": 0.07829800000001796,+ "p95Ms": 0.1582180000000335,+ "maxMs": 2.4970189999999093+ },+ "cold": {+ "p50Ms": 60.09458799999993,+ "p95Ms": 67.15132600000004,+ "maxMs": 82.73295499999995+ },+ "rssBytes": 40108032+ },+ {+ "pair": 1,+ "hot": {+ "p50Ms": 0.07857799999965209,+ "p95Ms": 0.13365100000009988,+ "maxMs": 3.1908830000002126+ },+ "cold": {+ "p50Ms": 58.721712999999,+ "p95Ms": 65.59099799999967,+ "maxMs": 66.20356299999912+ },+ "rssBytes": 39976960+ },+ {+ "pair": 2,+ "hot": {+ "p50Ms": 0.07846799999970244,+ "p95Ms": 0.14697199999864097,+ "maxMs": 3.2146300000003976+ },+ "cold": {+ "p50Ms": 56.40588399999979,+ "p95Ms": 62.27086600000075,+ "maxMs": 64.56847600000037+ },+ "rssBytes": 39976960+ },+ {+ "pair": 3,+ "hot": {+ "p50Ms": 0.07862799999929848,+ "p95Ms": 0.12908499999684864,+ "maxMs": 2.595773000000918+ },+ "cold": {+ "p50Ms": 56.496171000002505,+ "p95Ms": 61.56686500000069,+ "maxMs": 63.52171299999827+ },+ "rssBytes": 39976960+ },+ {+ "pair": 4,+ "hot": {+ "p50Ms": 0.07857800000056159,+ "p95Ms": 0.11362200000075973,+ "maxMs": 3.615970999999263+ },+ "cold": {+ "p50Ms": 56.157061000001704,+ "p95Ms": 59.52665699999852,+ "maxMs": 63.03953399999955+ },+ "rssBytes": 39976960+ },+ {+ "pair": 5,+ "hot": {+ "p50Ms": 0.07885900000110269,+ "p95Ms": 0.12179299999843352,+ "maxMs": 2.41507200000342+ },+ "cold": {+ "p50Ms": 59.719207999998616,+ "p95Ms": 70.95469899999807,+ "maxMs": 85.01844100000017+ },+ "rssBytes": 39976960+ },+ {+ "pair": 6,+ "hot": {+ "p50Ms": 0.07916000000113854,+ "p95Ms": 0.13390199999776087,+ "maxMs": 3.1793890000008105+ },+ "cold": {+ "p50Ms": 60.57355599999937,+ "p95Ms": 67.18024299999888,+ "maxMs": 73.57634400000097+ },+ "rssBytes": 40108032+ },+ {+ "pair": 7,+ "hot": {+ "p50Ms": 0.0787789999958477,+ "p95Ms": 0.11498399999982212,+ "maxMs": 2.096875999995973+ },+ "cold": {+ "p50Ms": 60.348517999998876,+ "p95Ms": 66.2843969999958,+ "maxMs": 66.82093100000202+ },+ "rssBytes": 39976960+ },+ {+ "pair": 8,+ "hot": {+ "p50Ms": 0.07898900000145659,+ "p95Ms": 0.11351100000320002,+ "maxMs": 1.119118000002345+ },+ "cold": {+ "p50Ms": 58.180186999998114,+ "p95Ms": 69.00580800000171,+ "maxMs": 73.97517200000584+ },+ "rssBytes": 39976960+ },+ {+ "pair": 9,+ "hot": {+ "p50Ms": 0.0787689999997383,+ "p95Ms": 0.12075200000253972,+ "maxMs": 1.514233999994758+ },+ "cold": {+ "p50Ms": 58.19894999999815,+ "p95Ms": 60.94138099999691,+ "maxMs": 62.39702099999704+ },+ "rssBytes": 39845888+ },+ {+ "pair": 10,+ "hot": {+ "p50Ms": 0.08065199999691686,+ "p95Ms": 0.14515899999969406,+ "maxMs": 2.812612000001536+ },+ "cold": {+ "p50Ms": 61.95117800000298,+ "p95Ms": 70.65781000000425,+ "maxMs": 75.84767800000554+ },+ "rssBytes": 40108032+ },+ {+ "pair": 11,+ "hot": {+ "p50Ms": 0.07912899999791989,+ "p95Ms": 0.146351000003051,+ "maxMs": 1.6679159999985131+ },+ "cold": {+ "p50Ms": 59.251374999999825,+ "p95Ms": 72.4842159999971,+ "maxMs": 80.97192899999936+ },+ "rssBytes": 39976960+ },+ {+ "pair": 12,+ "hot": {+ "p50Ms": 0.07935900000302354,+ "p95Ms": 0.14096199999767123,+ "maxMs": 1.0998499999986961+ },+ "cold": {+ "p50Ms": 60.49111599999742,+ "p95Ms": 64.17300899999827,+ "maxMs": 67.44247100000212+ },+ "rssBytes": 40239104+ },+ {+ "pair": 13,+ "hot": {+ "p50Ms": 0.07949999999982538,+ "p95Ms": 0.13854799999535317,+ "maxMs": 2.1440640000000712+ },+ "cold": {+ "p50Ms": 59.74794799999654,+ "p95Ms": 65.60776500000065,+ "maxMs": 67.50651999999536+ },+ "rssBytes": 40108032+ },+ {+ "pair": 14,+ "hot": {+ "p50Ms": 0.07983100000274135,+ "p95Ms": 0.14397699999972247,+ "maxMs": 3.442432000003464+ },+ "cold": {+ "p50Ms": 61.7351090000011,+ "p95Ms": 70.28328699999838,+ "maxMs": 79.00864800000272+ },+ "rssBytes": 40108032+ }+ ],+ "candidate": [+ {+ "pair": 0,+ "hot": {+ "p50Ms": 0.09069700000009107,+ "p95Ms": 0.18544999999994616,+ "maxMs": 2.5909590000001117+ },+ "cold": {+ "p50Ms": 59.03232799999978,+ "p95Ms": 65.61206200000015,+ "maxMs": 68.59447199999977+ },+ "rssBytes": 40239104+ },+ {+ "pair": 1,+ "hot": {+ "p50Ms": 0.09094600000025821,+ "p95Ms": 0.18210499999986496,+ "maxMs": 4.174164000000019+ },+ "cold": {+ "p50Ms": 60.23158299999977,+ "p95Ms": 76.7214240000003,+ "maxMs": 78.55946099999983+ },+ "rssBytes": 39976960+ },+ {+ "pair": 2,+ "hot": {+ "p50Ms": 0.08879300000080548,+ "p95Ms": 0.16084199999932025,+ "maxMs": 1.740814999999202+ },+ "cold": {+ "p50Ms": 59.25101099999847,+ "p95Ms": 65.57913299999927,+ "maxMs": 73.54857000000084+ },+ "rssBytes": 40108032+ },+ {+ "pair": 3,+ "hot": {+ "p50Ms": 0.08929400000124588,+ "p95Ms": 0.17173900000125286,+ "maxMs": 1.1591740000003483+ },+ "cold": {+ "p50Ms": 58.891552999999476,+ "p95Ms": 65.20447899999999,+ "maxMs": 65.55479800000103+ },+ "rssBytes": 39976960+ },+ {+ "pair": 4,+ "hot": {+ "p50Ms": 0.09003600000141887,+ "p95Ms": 0.1786790000005567,+ "maxMs": 1.3934010000011767+ },+ "cold": {+ "p50Ms": 59.73071499999787,+ "p95Ms": 66.92889899999864,+ "maxMs": 69.50185799999963+ },+ "rssBytes": 40108032+ },+ {+ "pair": 5,+ "hot": {+ "p50Ms": 0.08990500000072643,+ "p95Ms": 0.13865900000018883,+ "maxMs": 1.6249099999986356+ },+ "cold": {+ "p50Ms": 58.2477579999977,+ "p95Ms": 66.3452879999968,+ "maxMs": 67.7671279999995+ },+ "rssBytes": 40108032+ },+ {+ "pair": 6,+ "hot": {+ "p50Ms": 0.08999600000242935,+ "p95Ms": 0.16990699999951175,+ "maxMs": 7.246950000000652+ },+ "cold": {+ "p50Ms": 59.78370800000266,+ "p95Ms": 68.61635599999863,+ "maxMs": 73.99606199999835+ },+ "rssBytes": 40108032+ },+ {+ "pair": 7,+ "hot": {+ "p50Ms": 0.08988500000123167,+ "p95Ms": 0.16695099999924423,+ "maxMs": 1.2857589999985066+ },+ "cold": {+ "p50Ms": 61.638294999997015,+ "p95Ms": 66.90604700000404,+ "maxMs": 67.80749099999957+ },+ "rssBytes": 40108032+ },+ {+ "pair": 8,+ "hot": {+ "p50Ms": 0.08974600000510691,+ "p95Ms": 0.12813399999868125,+ "maxMs": 1.9490100000039092+ },+ "cold": {+ "p50Ms": 58.63375200000155,+ "p95Ms": 65.77171899999666,+ "maxMs": 72.06060500000603+ },+ "rssBytes": 39845888+ },+ {+ "pair": 9,+ "hot": {+ "p50Ms": 0.08939499999542022,+ "p95Ms": 0.12315600000147242,+ "maxMs": 1.9656749999994645+ },+ "cold": {+ "p50Ms": 59.898419999997714,+ "p95Ms": 68.33871599999839,+ "maxMs": 74.5328789999985+ },+ "rssBytes": 40108032+ },+ {+ "pair": 10,+ "hot": {+ "p50Ms": 0.10770300000149291,+ "p95Ms": 0.18338699999731034,+ "maxMs": 2.2291330000007292+ },+ "cold": {+ "p50Ms": 61.15316999999777,+ "p95Ms": 68.40321000000404,+ "maxMs": 72.53870199999801+ },+ "rssBytes": 39976960+ },+ {+ "pair": 11,+ "hot": {+ "p50Ms": 0.0895849999942584,+ "p95Ms": 0.12595000000146683,+ "maxMs": 1.6352760000008857+ },+ "cold": {+ "p50Ms": 57.32905100000062,+ "p95Ms": 63.90412899999501,+ "maxMs": 64.8627639999977+ },+ "rssBytes": 39976960+ },+ {+ "pair": 12,+ "hot": {+ "p50Ms": 0.09007600000040838,+ "p95Ms": 0.15343099999881815,+ "maxMs": 3.4101280000031693+ },+ "cold": {+ "p50Ms": 58.53407799999695,+ "p95Ms": 62.44271000000299,+ "maxMs": 69.35012899999856+ },+ "rssBytes": 40108032+ },+ {+ "pair": 13,+ "hot": {+ "p50Ms": 0.09051699999690754,+ "p95Ms": 0.15944199999648845,+ "maxMs": 3.285585000005085+ },+ "cold": {+ "p50Ms": 59.712093000001914,+ "p95Ms": 65.43425399999978,+ "maxMs": 66.71044800000527+ },+ "rssBytes": 40108032+ },+ {+ "pair": 14,+ "hot": {+ "p50Ms": 0.09065600000030827,+ "p95Ms": 0.1295749999990221,+ "maxMs": 4.211842999997316+ },+ "cold": {+ "p50Ms": 60.56668299999728,+ "p95Ms": 67.47013099999458,+ "maxMs": 68.6809879999928+ },+ "rssBytes": 39976960+ }+ ]+ },+ "baseline": {+ "hotP95Ms": 0.13390199999776087,+ "coldP95Ms": 66.2843969999958,+ "rssBytes": 39976960+ },+ "candidate": {+ "hotP95Ms": 0.16084199999932025,+ "coldP95Ms": 66.3452879999968,+ "rssBytes": 40108032+ },+ "delta": {+ "hotP95Ms": 0.026940000001559383,+ "hotP95Percent": 20.119191649123902,+ "coldP95Ms": 0.06089100000099279,+ "coldP95Percent": 0.09186324799936862,+ "rssBytes": 131072+ },+ "measurementValid": true,+ "gates": {+ "hot": false,+ "cold": true,+ "rss": true+ }+}diff --git a/docs/self-improvement/2026-09-25-evidence.json b/docs/self-improvement/2026-09-25-evidence.jsonnew file mode 100644index 0000000..1b3fab1--- /dev/null+++ b/docs/self-improvement/2026-09-25-evidence.json@@ -0,0 +1,260 @@+{+ "date": "2026-09-25",+ "baseline": "211f5751515446a3d994d43adacc5d5fba7d18dc",+ "candidate": {+ "id": "reviewed-upstream-evidence-refresh",+ "commit": "71085634192d721beab7fc4b4b9eabc1b727097f",+ "version": "0.10.1-local-candidate",+ "state": "rejected",+ "promotion": "not-pushed",+ "reason": "The frozen hot-search p95 gate failed: 20.119191649123902% regression exceeded the 20% ceiling, although the absolute increase was 0.026940000001559383 ms and passed the separate 0.05 ms ceiling.",+ "snapshotId": "sha256:d27c8abb1ab9f48689d2bde29ee4a88c4165d6cf5d7932b890ba410405211a62",+ "heldOut": {+ "baseline": "1/6",+ "candidate": "6/6",+ "runnerSha256": "a51ab5d5510fbf090f191c6a9ff1718c960d6f4a240e393523fa59cd38eaf909",+ "evaluatorChangedAfterCandidateScoring": false+ },+ "tests": {+ "groups": 17,+ "node22": "pass after removing proxy variables that caused an experimental runtime warning on stderr",+ "node24": "pass"+ },+ "metaharness": {+ "package": "[email protected]",+ "hard": "6/6",+ "harnessFit": 60,+ "compileConfidence": 65,+ "taskCoverage": 84,+ "toolSafety": 90,+ "memoryUsefulness": 38,+ "scaffoldReady": true,+ "proofBoundary": "Heuristic constraints and scores are not proof of improvement."+ },+ "performance": {+ "runnerSha256": "8249c7bdc011fc3a1b1c1581aef7d56c3b5fe642a9caaeca84fff4c0d9cd57bb",+ "rawSha256": "648c37403067da79d4f112ff2d70e99b3d6a224b887f9bf81d5a62048749e350",+ "seed": 1729,+ "seedUsage": "metadata only; fixed alternating order and no RNG",+ "pairs": 15,+ "hotCallsPerArmPerPair": 10000,+ "coldStartsPerArmPerPair": 20,+ "measurementValid": true,+ "baseline": {+ "hotP95Ms": 0.13390199999776087,+ "coldP95Ms": 66.2843969999958,+ "rssBytes": 39976960+ },+ "candidate": {+ "hotP95Ms": 0.16084199999932025,+ "coldP95Ms": 66.3452879999968,+ "rssBytes": 40108032+ },+ "delta": {+ "hotP95Ms": 0.026940000001559383,+ "hotP95Percent": 20.119191649123902,+ "coldP95Ms": 0.06089100000099279,+ "coldP95Percent": 0.09186324799936862,+ "rssBytes": 131072+ },+ "gates": {+ "hot": false,+ "cold": true,+ "rss": true+ },+ "raw": "2026-09-25-benchmarks.json"+ },+ "package": {+ "compressedBytes": 31437,+ "unpackedBytes": 108038,+ "files": 18,+ "dependencyClosureEntries": 94,+ "dependencyClosureSha256": "084b16e3f9e1d06fe293bcaef9b776c652ace9db31a3f95b387e3e010fb93bec",+ "closureUnchanged": true+ },+ "authority": {+ "mcpTools": 6,+ "mcpResources": 3,+ "skills": 4,+ "marketplaces": 2,+ "expanded": false+ }+ },+ "upstream": {+ "observedAt": "2026-09-25T10:15:00Z",+ "sources": [+ {+ "id": "ruflo",+ "revision": "88955d9fa9c60836a3f9a3d0e47fcabdc5af4ff5",+ "release": "v3.45.0",+ "license": "MIT",+ "compatibility": "Node >=20",+ "changed": true+ },+ {+ "id": "metaharness",+ "revision": "d5833dc6512ac1adeeef91a331c29055cd8a4dbb",+ "release": "v0.4.4; scorer package 0.4.16",+ "license": "MIT",+ "compatibility": "Node >=20; Rust 1.85",+ "changed": false+ },+ {+ "id": "ruvector",+ "revision": "5356a84e2f784a33fa497da2e73440d469eb5542",+ "release": "GitHub ruvector-v0.2.40; npm ruvector 0.3.3",+ "license": "MIT",+ "compatibility": "Node >=20 for ruvector; related packages >=18; Rust 1.77",+ "changed": false+ },+ {+ "id": "agentdb",+ "revision": "6cdefe451f6afe7395bc88d3587a0225d161089b",+ "release": "npm 3.0.0-alpha.20",+ "license": "MIT",+ "compatibility": "Node >=18",+ "changed": false+ },+ {+ "id": "dream-machine",+ "revision": "aa931caad5dd0108253645bba0ab1481ad7da0ee",+ "release": "v0.1.0",+ "license": "MIT",+ "compatibility": "Node ^22.13 or ^24",+ "changed": false+ },+ {+ "id": "autogenous",+ "revision": "905aa6cbe213392f8b3cab5d4f17bc3a48e0a509",+ "release": "radio-moe-v0.3.1",+ "license": "MIT",+ "compatibility": "Rust 1.74",+ "changed": false+ },+ {+ "id": "rvm",+ "revision": "0973d77e5a9e99065376e8ef772a4a1d16e88c75",+ "release": "v1.7.0",+ "license": "manifest declares MIT OR Apache-2.0; root license texts absent",+ "compatibility": "Rust 1.77",+ "changed": false+ }+ ],+ "candidateRecords": [+ "ruflo-honest-learning-counts",+ "ruflo-router-benchmark-no-promotion",+ "ruvector-mcp-intelligence-durable-save",+ "ruvector-graph-metadata-roundtrip",+ "ruvector-quantize-fail-closed",+ "ruvector-filter-recursive-serde-fix"+ ],+ "newlyReviewedNotInCandidate": [+ {+ "id": "ruflo-memory-exact-pin-and-stale-doctor",+ "revision": "88955d9fa9c60836a3f9a3d0e47fcabdc5af4ff5",+ "limitations": "No formal PR review; one unexplained CI failure followed by a same-SHA rerun; 99 build-output witness entries were carried forward; the wrapper still permits a stale CLI range."+ },+ {+ "id": "ruflo-init-test-home-isolation",+ "revision": "c815ccdeed7fa94b5bb1e8337d9811b70e610de6",+ "limitations": "Test-only safety isolation; no runtime capability or formal PR review."+ }+ ],+ "incorporation": "none because the candidate failed its frozen performance gate",+ "xRuvIo": {+ "gateway": "0.7.2",+ "mcp": "/mcp",+ "chatgptMcp": "/chatgpt/mcp",+ "oauthScopes": ["swarm:read", "swarm:publish"],+ "status": "unchanged"+ }+ },+ "security": {+ "rufloCli": "3.25.6",+ "deepAll": {+ "critical": 0,+ "high": 0,+ "medium": 0,+ "low": 0+ },+ "npmAudit": {+ "critical": 0,+ "high": 0,+ "total": 0,+ "productionDependencies": 95+ },+ "limitation": "The scanner reported no known issues but its coverage is limited; this is not a complete-security claim."+ },+ "memory": {+ "shippedEligibleRecords": 59,+ "candidateEligibleRecords": 65,+ "reviewedEligibleRecordsIncludingNewRuflo": 67,+ "gate": 100,+ "exactBaseline": "implemented",+ "vectorRetrieval": "deferred",+ "wasmAcceleration": "not implemented",+ "gnnTraining": "not implemented",+ "persistentProductLearning": "not implemented",+ "coordinationMemoryWrite": "skipped because the candidate failed its frozen acceptance gate"+ },+ "ruos": {+ "status": "INCONCLUSIVE",+ "liveProbeRequired": false,+ "liveProbeIssued": false,+ "desktopHeartbeat": null,+ "trustedEvaluatorRevision": "aa931caad5dd0108253645bba0ab1481ad7da0ee",+ "runbookSha256": "b10cb10844a6dbf045c80e0d814fd8c2c379e77eae007c4000cd37df4eabb2eb",+ "preflightSha256": "e2643a4c0bf8271a297c36c74c830dd285938f8a534da53f604f7983c22330dd",+ "evaluationSha256": "330189c4f31af1ab1dae37b97269cbc51834884f5a357415fb364c67932bb5eb",+ "boundary": "No live desktop, tenant, authoritative completion, screenshot or customer journey was established; a local data refresh did not require a desktop probe."+ },+ "forecast": {+ "type": "prediction",+ "confidence": 0.81,+ "horizon": "2027-01-23",+ "prediction": "Agent-memory and retrieval promotion suites will require state-conditioned, set-level evidence sufficiency, so individually relevant but collectively incomplete top-k results cannot pass.",+ "falsifiableCriterion": "At least two primary benchmarks or reviewed production harnesses first published or materially revised after 2026-09-25 must include grouped evidence requirements, a held-out split, an exact set-completeness or coverage metric and a downstream missing-group ablation; fewer than two falsifies the prediction.",+ "priorForecastExpired": false,+ "sources": [+ "https://arxiv.org/abs/2609.20050",+ "https://github.com/LordTARN1SHED/SERBench",+ "https://arxiv.org/abs/2608.10108",+ "https://arxiv.org/abs/2607.26072",+ "https://github.com/microsoft/STATE-Bench/blob/main/docs/AGENT_LEARNING_TRACK.md"+ ]+ },+ "delivery": {+ "sourceChange": "not promoted",+ "candidateCi": "not run because the frozen local performance gate rejected before promotion",+ "reportOnlyValidation": "Node 22 and 24: validator and 16/16 shipped-baseline test groups pass; all report JSON parses and git diff check passes",+ "reportCommit": null,+ "reportCi": null,+ "sites": {+ "projectId": "appgprj_6aa529f096ac8191b708898ad9e6ef2d",+ "status": "pending",+ "audience": "owner-only",+ "url": "https://ruvnet-harness-evidence.ruv.chatgpt.site"+ }+ },+ "independentReview": {+ "decision": "reject",+ "reason": "The valid hot-search measurement exceeded the frozen relative ceiling; passing absolute, cold and RSS gates cannot override the conjunctive rule.",+ "verified": [+ "candidate is exactly bound to the frozen-plan parent",+ "runner hashes match the plan",+ "rejected catalog parser is absent",+ "tools, MCP runtime code, skills and dependencies are unchanged",+ "origin/main remains 211f575 and the candidate is not promoted"+ ]+ },+ "limitations": [+ "The candidate failed a conjunctive threshold by 0.119 percentage points and was not promoted; the threshold was not reinterpreted.",+ "Upstream results were reviewed as public evidence, not independently reproduced in RuVnet.",+ "The initial Node 22 npx run emitted an experimental proxy warning on stderr; the same exact tests passed with proxy variables removed.",+ "Held-out groups 4 and 5 inspect raw snapshot records and boundaries; exact-search journeys are exercised by groups 2 and 3 plus the regression suite.",+ "Tool, resource and marketplace counts come partly from the separate regression and validator suites rather than solely from the held-out runner.",+ "No live ruOS desktop evidence exists.",+ "No vector, WASM acceleration, GNN or autonomous-learning value is claimed."+ ],+ "nextHypothesis": "Evaluate set-complete exact retrieval over a larger reviewed corpus, retaining explicit negative-result and provenance groups; do not introduce RuVector or GNN complexity before the 100-record gate."+}diff --git a/docs/self-improvement/2026-09-25-plan.json b/docs/self-improvement/2026-09-25-plan.jsonnew file mode 100644index 0000000..c5c7bbf--- /dev/null+++ b/docs/self-improvement/2026-09-25-plan.json@@ -0,0 +1,119 @@+{+ "date": "2026-09-25",+ "status": "frozen-before-candidate",+ "baseline": "211f5751515446a3d994d43adacc5d5fba7d18dc",+ "target": "refs/heads/main",+ "acceptedButUnintegrated": [],+ "priorRejectedCandidate": {+ "id": "cli-catalog-named-limit-parity",+ "decision": "not-retried",+ "reason": "The 2026-09-24 candidate failed its frozen hot-p95 gate; no rejected parser code is included in this data-only experiment."+ },+ "candidate": {+ "id": "reviewed-upstream-evidence-refresh",+ "version": "0.10.1",+ "hypothesis": "Adding the six independently reviewed Ruflo and RuVector records to the read-only provenance snapshot will improve held-out exact evidence retrieval from 1/6 to 6/6 while preserving all 48 prior records, default behavior and authority.",+ "scope": [+ "reviewed upstream data and current source-head metadata",+ "patch version metadata",+ "tests and documentation"+ ],+ "excluded": [+ "the rejected catalog limit parser",+ "upstream runtime code",+ "new vector, WASM, GNN or autonomous-learning functionality",+ "the two newly reviewed Ruflo 3.45.0 records, which remain reviewed-not-incorporated for a separate bounded experiment"+ ],+ "authority": "read-only, offline, no new tool, resource, network, shell, write, credential or federation capability"+ },+ "heldOut": {+ "runner": "../heldout-20260925.mjs",+ "sha256": "a51ab5d5510fbf090f191c6a9ff1718c960d6f4a240e393523fa59cd38eaf909",+ "baselinePassed": 1,+ "total": 6,+ "groups": [+ "the snapshot contains 54 unique records, current pinned source heads, a fixed observation time and a new deterministic identity",+ "honest learning-count reporting is exact-search retrievable and explicitly bounded to reporting integrity",+ "the router benchmark is a negative result retaining 197 prompts, 113 held-out prompts, no winner and an unchanged default",+ "durable intelligence save and graph-metadata round-trip evidence retain atomic, quarantine and reserved-key boundaries",+ "quantize fail-closed and recursive WASM filter serialization retain unimplemented-writer and no-acceleration boundaries",+ "all 48 prior records remain byte-canonical and ordered, least-privilege surfaces remain fixed, CLI retrieval works and the old snapshot fails closed"+ ]+ },+ "acceptance": {+ "heldOut": "6/6 and strictly better than the frozen 1/6 baseline; evaluator must not change after candidate scoring",+ "node": [22, 24],+ "existingTests": "all pass",+ "marketplaces": 2,+ "skills": 4,+ "mcpTools": 6,+ "mcpResources": 3,+ "metaharness": "[email protected], 6/6 hard constraints; heuristic status is not proof",+ "performance": {+ "runner": "../benchmark-20260925.mjs",+ "sha256": "8249c7bdc011fc3a1b1c1581aef7d56c3b5fe642a9caaeca84fff4c0d9cd57bb",+ "seed": 1729,+ "seedUsage": "metadata only; fixed alternating order and no RNG",+ "pairs": 15,+ "hotCallsPerArmPerPair": 10000,+ "coldStartsPerArmPerPair": 20,+ "workload": "memory quantize graph metadata learning router",+ "measurementValidity": "baseline hot p95 must be at least 0.01 ms; otherwise INCONCLUSIVE and no promotion",+ "hotP95MaximumRegression": "both <=20% and <=0.05 ms absolute",+ "coldP95MaximumRegression": "both <=20% and <=5 ms absolute",+ "rssMaximumRegressionBytes": 5242880+ },+ "package": "normalized 94-entry dependency closure byte-identical, no new dependency, valid package, 18 files and compressed archive below 40000 bytes",+ "security": "npm audit has no high/critical finding; pinned Ruflo code, dependency, secret and STRIDE scans complete with coverage limitations disclosed",+ "promotion": "candidate bound to an exact commit, independent review, candidate CI, exact main-head recheck, fast-forward only and post-promotion CI"+ },+ "upstreamCutoff": {+ "observedAt": "2026-09-25T10:15:00Z",+ "ruflo": "88955d9fa9c60836a3f9a3d0e47fcabdc5af4ff5",+ "metaharness": "d5833dc6512ac1adeeef91a331c29055cd8a4dbb",+ "ruvector": "5356a84e2f784a33fa497da2e73440d469eb5542",+ "agentdb": "6cdefe451f6afe7395bc88d3587a0225d161089b",+ "dreamMachine": "aa931caad5dd0108253645bba0ab1481ad7da0ee",+ "autogenous": "905aa6cbe213392f8b3cab5d4f17bc3a48e0a509",+ "rvm": "0973d77e5a9e99065376e8ef772a4a1d16e88c75"+ },+ "ruvectorWorkstream": {+ "shippedEligibleRecordsBeforeRefresh": 59,+ "reviewedEligibleRecordsAfterRefresh": 65,+ "minimumRecords": 100,+ "baseline": "exact lexical retrieval",+ "decision": "defer product vector retrieval, WASM acceleration and GNN learning below the frozen corpus gate",+ "futureArms": ["exact", "vector-only", "fusion", "GNN rerank", "shuffled-graph control"],+ "futureMinimumGain": "10 percentage points held-out downstream task success"+ },+ "ruos": {+ "runNonce": "c3d6e616-8c19-482d-8558-0285533471b0",+ "status": "INCONCLUSIVE",+ "liveProbeRequired": false,+ "reason": "The candidate changes reviewed local data only and does not affect a desktop journey, desktop control, MCP UI, OAuth or graphical performance.",+ "candidateDesktop": "59fc0c7fcdd873c1eba47092dfee29e2",+ "candidateDesktopHeartbeat": null,+ "evaluatorRevision": "aa931caad5dd0108253645bba0ab1481ad7da0ee",+ "runbookSha256": "b10cb10844a6dbf045c80e0d814fd8c2c379e77eae007c4000cd37df4eabb2eb",+ "evaluatorSha256": "330189c4f31af1ab1dae37b97269cbc51834884f5a357415fb364c67932bb5eb",+ "preflightSha256": "e2643a4c0bf8271a297c36c74c830dd285938f8a534da53f604f7983c22330dd",+ "expectedTenant": null,+ "assertionIds": [],+ "minimumSamples": 0+ },+ "forecast": {+ "type": "prediction",+ "prediction": "Agent-memory and retrieval promotion suites will require state-conditioned, set-level evidence sufficiency, so individually relevant but collectively incomplete top-k results cannot pass.",+ "confidence": 0.81,+ "horizon": "2027-01-23",+ "falsifiableCriterion": "At least two primary benchmarks or reviewed production harnesses first published or materially revised after 2026-09-25 must include grouped evidence requirements, a held-out split, an exact set-completeness or coverage metric and a downstream missing-group ablation; fewer than two falsifies the prediction.",+ "priorForecastExpired": false,+ "sources": [+ "https://arxiv.org/abs/2609.20050",+ "https://github.com/LordTARN1SHED/SERBench",+ "https://arxiv.org/abs/2608.10108",+ "https://arxiv.org/abs/2607.26072",+ "https://github.com/microsoft/STATE-Bench/blob/main/docs/AGENT_LEARNING_TRACK.md"+ ]+ }+}diff --git a/docs/self-improvement/2026-09-25.md b/docs/self-improvement/2026-09-25.mdnew file mode 100644index 0000000..0bea846--- /dev/null+++ b/docs/self-improvement/2026-09-25.md@@ -0,0 +1,41 @@+# RuVnet harness improvement — 2026-09-25++## Decision++Rejected the data-only `0.10.1` candidate and did not promote it. The candidate would have added six previously reviewed Ruflo and RuVector records to the content-addressed, read-only snapshot while preserving all 48 prior records. It improved the frozen held-out evaluation from 1/6 to 6/6, but failed the frozen hot-search latency gate. The shipped entry point remains `0.10.0` at [`211f575`](https://github.com/ruvnet/ruvnet/commit/211f5751515446a3d994d43adacc5d5fba7d18dc).++Yesterday's rejected catalog-limit parser was not retried or included. There are no accepted-but-unintegrated candidates or unmerged remote branches. Candidate commit `71085634192d721beab7fc4b4b9eabc1b727097f` is retained only in the local evaluation worktree and was not pushed.++## Upstream review++Ruflo advanced to [`88955d9`](https://github.com/ruvnet/ruflo/commit/88955d9fa9c60836a3f9a3d0e47fcabdc5af4ff5), release [3.45.0](https://github.com/ruvnet/ruflo/releases/tag/v3.45.0), under MIT with Node 20 or newer. The release pins both `@claude-flow/memory` dependency declarations to `3.0.0-alpha.25` and adds a warning-only loaded-version check. It also isolates the init test's home and Codex configuration. These are established compatibility and test-safety changes, not evidence of memory quality. Neither PR had a formal review; the release records a same-SHA CI rerun after one unexplained failure, 99 carried-forward build-output witness entries, an unfixed wrapper range that can retain an older CLI, and an unfixed old MCP shell helper. The two new records were reviewed but kept out of this already bounded candidate.++The six September 24 records remain valid ancestors of current Ruflo and RuVector main: honest learning-count reporting, a 197-prompt/113-held-out router benchmark with no promoted winner, atomic intelligence persistence with corrupt-state quarantine, graph-metadata round trips, fail-closed quantization while its writer remains unimplemented, and recursive filter serialization restoring WASM filter compatibility without an acceleration claim. RuVector remained at [`5356a84`](https://github.com/ruvnet/RuVector/commit/5356a84e2f784a33fa497da2e73440d469eb5542). MetaHarness, AgentDB, Dream Machine, Autogenous and RVM were unchanged. Exact releases, compatibility and licenses are in the [machine-readable evidence](2026-09-25-evidence.json). [x.ruv.io](https://x.ruv.io/) remained gateway 0.7.2 with separate `/mcp` and `/chatgpt/mcp` routes and read/publish scopes. All fetched material was treated as untrusted evidence.++## MetaHarness loop and measurements++The [frozen plan](2026-09-25-plan.json) and external runner hash fixed the hypothesis, six held-out groups and thresholds before candidate implementation. Baseline scored 1/6. The candidate scored 6/6: exact provenance and source heads, bounded learning-report evidence, router abstention evidence, durable-save and graph-metadata record boundaries, quantize/filter limitations, and preservation of prior records plus the fixed authority surface. Groups 4 and 5 inspect raw snapshot records rather than independent end-to-end journeys; exact search is exercised by groups 2 and 3 and the regression suite. One candidate wording was tightened after the unchanged evaluator rejected the searchable phrase “RuVnet learning” even in a negated sentence; no evaluator or threshold changed.++All 17 candidate test groups and both marketplace validators passed on Node 22 and 24. The first Node 22 invocation inherited proxy variables and emitted an experimental runtime warning on stderr, causing the negative CLI assertion to fail; the same exact suite passed after removing those environmental variables. Four skills, six MCP tools and three resources remained fixed. MetaHarness 0.4.16 retained 6/6 hard constraints; its heuristic scores are not proof of improvement. The package contained 18 files, compressed to 31,437 bytes, and its normalized 94-entry dependency closure was byte-identical to baseline.++Fifteen deterministic alternating pairs used 10,000 exact-search calls and 20 cold CLI starts per arm per pair. Baseline hot p95 was 0.133902 ms and candidate hot p95 was 0.160842 ms: +20.119%, +0.026940 ms. The frozen rule required both at most +20% relative and +0.05 ms absolute, so the 0.119 percentage-point relative miss rejects the candidate. Cold p95 rose 0.0919% (+0.0609 ms); RSS rose 131,072 bytes. Both passed. [Raw paired measurements](2026-09-25-benchmarks.json) retain every sample. The threshold was not reinterpreted and no outcome-driven rerun was made.++Pinned Ruflo 3.25.6 deep scanning and npm audit reported zero known high or critical findings. Scanner coverage is limited, so this is not a complete-security claim. An independent review confirmed the rejection: passing the absolute, cold and RSS gates cannot override the conjunctive hot-p95 failure. Since the local gate failed, no candidate branch was pushed and no candidate CI was requested.++## ruOS, memory and learning++ruOS remains **INCONCLUSIVE**. Dream Machine main and its reviewed runbook/verifier were unchanged at `aa931ca`; all ten known desktops still lacked a heartbeat. This local data-only candidate did not affect a desktop journey, desktop control, MCP UI, OAuth or graphical performance, so the frozen policy did not require a live probe. No command or screenshot request was issued, and no tenant or authoritative completion was inferred.++The shipped corpus remains 59 eligible catalog-plus-change records. The rejected candidate would have raised it to 65; including today's two newly reviewed Ruflo records yields 67 reviewed records, still below the frozen 100-record vector/GNN gate. Exact lexical retrieval remains the only shipped baseline. Product RuVector retrieval, WASM acceleration, GNN training and persistent autonomous learning remain unimplemented and unclaimed. No coordination-memory pattern was stored because the candidate failed.++## Forecast++Prediction, 81% confidence by 2027-01-23: agent-memory and retrieval promotion suites will require state-conditioned, set-level evidence sufficiency, so individually relevant but collectively incomplete top-k results cannot pass. It passes only if at least two primary benchmarks or reviewed production harnesses first published or materially revised after 2026-09-25 include grouped evidence requirements, a held-out split, an exact completeness or coverage metric, and a downstream missing-group ablation. Fewer than two falsifies it. No prior forecast horizon expired today.++Established results are narrower. [SERBench](https://arxiv.org/abs/2609.20050) scores whether a retrieved set covers every fact required by a captured agent state and includes a missing-group downstream ablation; its [official implementation](https://github.com/LordTARN1SHED/SERBench) separates public calibration from held-out evaluation. [MESA](https://arxiv.org/abs/2608.10108) reports task-adaptive composition of complementary memory structures. [IFCMemoryBench](https://arxiv.org/abs/2607.26072) and the official [STATE-Bench agent-learning track](https://github.com/microsoft/STATE-Bench/blob/main/docs/AGENT_LEARNING_TRACK.md) further motivate evaluation beyond isolated top-k relevance. These sources motivate the prediction; they do not prove future adoption or justify lowering the 100-record gate.++## Dashboard and next hypothesis++The existing owner-only [RuVnet Harness Evidence dashboard](https://ruvnet-harness-evidence.ruv.chatgpt.site) is updated in place with the rejection, paired metrics, upstream review, forecast and memory/ruOS boundaries.++Next hypothesis: evaluate set-complete exact retrieval over a larger reviewed corpus, retaining explicit provenance and negative-result groups. Do not introduce RuVector or GNN complexity before the 100-record gate.