Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* MEMORY_VERBS v1 — conformance fixtures (E2).
*
* The canonical, embedded fixture set the conformance runner executes against
* ANY MCP endpoint (default: gbrain's own server). Deterministic by
* construction: every check is SHAPE (required fields, enum validity, const),
* CONTRACT BEHAVIOR (provenance rejected, idempotent forget, miss→found:false),
* or ROUND-TRIP (remember → recall by entity, a plain indexed read — no
* embeddings, no ranking judgment). Ranking quality is BrainBench's job.
*
* These double as BrainBench seeds: `test/fixtures/memory-verbs/cases.json` is
* the generated data-file mirror (drift-guarded by the conformance test) so
* the eval suite imports the same cases without importing gbrain source.
*
* Stateful round-trips share a per-run marker (substituted for `{{marker}}`)
* and thread ids via `saveAs`/`{{id:<key>}}` substitution.
*/
export interface ConformanceCase {
name: string;
verb: 'recall' | 'remember' | 'entity' | 'synthesize' | 'forget' | 'context_pack' | 'delta';
/** `{{marker}}` and `{{id:<key>}}` substitute at run time. */
params: Record<string, unknown>;
/** Validate the (parsed) response body against RESPONSE_SCHEMAS[verb]. */
validateSchema?: boolean;
/** Expect an isError response with this protocol code. */
expectErrorCode?: string;
/** On error responses: suggestion must be non-empty (F-D mandate). */
expectSuggestion?: boolean;
/** Field equality/predicate checks on the parsed response body. */
expect?: Array<
| { path: string; equals: unknown }
| { path: string; oneOf: unknown[] }
| { path: string; type: 'string' | 'number' | 'boolean' | 'array' | 'object' }
| { path: string; gte: number }
| { path: string; lte: number }
| { path: string; nonEmptyString: true }
| { path: string; absentOrNotContains: string }
>;
/** Save a response field for later cases ({{id:<key>}}). */
saveAs?: { key: string; path: string };
/** Only run when the runner was invoked with --synthesize (cost gate). */
requiresSynthesizeFlag?: boolean;
/**
* Only run when the runner could seed the conformance entity PAGE (via
* put_page when the target exposes it — full surface). entity() resolves
* pages; on a verbs-only target with no seeding path these cases skip
* honestly instead of failing on a structurally absent page.
*/
requiresSeededEntity?: boolean;
}
export const CONFORMANCE_CASES: ConformanceCase[] = [
// ── remember: contract behavior ─────────────────────────────────────────
{
name: 'remember rejects missing provenance (provenance_required + suggestion)',
verb: 'remember',
params: { fact: 'conformance {{marker}} fact without provenance' },
expectErrorCode: 'invalid_params', // MCP-level required-param rejection
expectSuggestion: false, // transport-level validation may not carry one
},
{
name: 'remember rejects empty provenance (provenance_required + suggestion)',
verb: 'remember',
params: { fact: 'conformance {{marker}} fact empty provenance', provenance: ' ' },
expectErrorCode: 'provenance_required',
expectSuggestion: true,
},
{
name: 'remember rejects ISO-8601 duration ttl with a fix (P30D trap)',
verb: 'remember',
params: { fact: 'conformance {{marker}} ttl trap', provenance: 'conformance run', ttl: 'P30D' },
expectErrorCode: 'invalid_params',
expectSuggestion: true,
},
{
name: 'remember writes a fact (string id, enum status, echoed nulls)',
verb: 'remember',
params: {
fact: 'conformance {{marker}}: the protocol round-trip fact',
provenance: 'conformance run {{marker}}',
entity: 'people/conformance-{{marker}}',
kind: 'fact',
},
validateSchema: true,
expect: [
{ path: 'id', type: 'string' },
{ path: 'status', oneOf: ['inserted', 'duplicate', 'superseded'] },
{ path: 'protocol_version', equals: 1 },
],
saveAs: { key: 'fact1', path: 'id' },
},
{
name: 'remember with ttl returns ISO valid_until',
verb: 'remember',
params: {
fact: 'conformance {{marker}}: expiring fact',
provenance: 'conformance run {{marker}}',
entity: 'people/conformance-{{marker}}',
ttl: '30d',
},
validateSchema: true,
expect: [{ path: 'valid_until', type: 'string' }],
saveAs: { key: 'fact2', path: 'id' },
},
{
name: 'remember private fact (fence test setup)',
verb: 'remember',
params: {
fact: 'conformance {{marker}} PRIVATE-SENTINEL commitment',
provenance: 'conformance run {{marker}}',
entity: 'people/conformance-{{marker}}',
kind: 'commitment',
visibility: 'private',
},
validateSchema: true,
},
// ── recall: round-trip + superset + budget ──────────────────────────────
{
name: 'recall by entity round-trips the remembered fact (superset envelope)',
verb: 'recall',
params: { entity: 'people/conformance-{{marker}}' },
validateSchema: true,
expect: [
{ path: 'protocol_version', equals: 1 },
{ path: 'total', gte: 1 },
{ path: 'facts.0.fact_id', type: 'string' },
{ path: 'facts.0.provenance', type: 'string' },
],
},
{
name: 'recall with budget reports consistent budget meta',
verb: 'recall',
params: { entity: 'people/conformance-{{marker}}', budget_tokens: 10000 },
validateSchema: true,
expect: [
{ path: 'budget_tokens', equals: 10000 },
{ path: 'budget_used', gte: 0 },
{ path: 'budget_used', lte: 10000 },
{ path: 'dropped_count', gte: 0 },
],
},
{
name: 'recall with budget smaller than the first item drops everything',
verb: 'recall',
params: { entity: 'people/conformance-{{marker}}', budget_tokens: 1 },
validateSchema: true,
expect: [
{ path: 'total', equals: 0 },
{ path: 'dropped_count', gte: 1 },
],
},
{
name: 'recall with query returns the search arm (degradation allowed, never an error)',
verb: 'recall',
params: { query: 'conformance {{marker}} protocol round-trip', budget_tokens: 8000 },
validateSchema: true,
expect: [{ path: 'results', type: 'array' }],
},
// ── entity: hit, miss, privacy fence ────────────────────────────────────
{
name: 'entity resolves the round-trip entity to a card',
verb: 'entity',
params: { name: 'people/conformance-{{marker}}' },
validateSchema: true,
requiresSeededEntity: true,
expect: [
{ path: 'found', equals: true },
{ path: 'protocol_version', equals: 1 },
{ path: 'card.entity.slug', type: 'string' },
{ path: 'card.backlink_count', gte: 0 },
],
},
{
name: 'entity miss returns found:false + suggestions (never an error)',
verb: 'entity',
params: { name: 'zzz-no-such-entity-{{marker}}' },
validateSchema: true,
expect: [
{ path: 'found', equals: false },
{ path: 'suggestions', type: 'array' },
],
},
{
name: 'entity card never leaks private facts to remote callers (fence test)',
verb: 'entity',
params: { name: 'people/conformance-{{marker}}' },
requiresSeededEntity: true,
expect: [{ path: 'card.open_threads', absentOrNotContains: 'PRIVATE-SENTINEL' }],
},
{
name: 'entity without the required name param is invalid_params with a suggestion',
verb: 'entity',
params: {},
// Unlike the remember missing-provenance case above (expectSuggestion:
// false — a bare transport-level rejection is tolerated there), this case
// holds servers to the full protocol envelope: any server that answers
// with `error: "invalid_params"` JSON is speaking MEMORY_VERBS v1, and
// v1 mandates a POPULATED suggestion on every verb error (F-D).
expectErrorCode: 'invalid_params',
expectSuggestion: true,
},
// ── forget: idempotency + not_found ─────────────────────────────────────
{
name: 'forget expires the remembered fact',
verb: 'forget',
params: { id: '{{id:fact2}}', reason: 'conformance cleanup' },
validateSchema: true,
expect: [{ path: 'expired', equals: true }],
},
{
name: 'forget again is idempotent (expired:false, success)',
verb: 'forget',
params: { id: '{{id:fact2}}' },
validateSchema: true,
expect: [{ path: 'expired', equals: false }],
},
{
name: 'remember of the forgotten claim is refused invalid_params with a suggestion (a withdrawn object, when present, is schema-valid)',
verb: 'remember',
params: {
fact: 'conformance {{marker}}: expiring fact',
provenance: 'conformance run {{marker}}',
entity: 'people/conformance-{{marker}}',
},
expectErrorCode: 'invalid_params',
expectSuggestion: true,
},
{
name: 'forget unknown id returns not_found with a suggestion',
verb: 'forget',
params: { id: '999999999' },
expectErrorCode: 'not_found',
expectSuggestion: true,
},
// ── synthesize: cost-gated live call ────────────────────────────────────
{
name: 'synthesize answers (or reports unavailable) — cost-gated, pass --synthesize',
verb: 'synthesize',
params: { question: 'What do we know about conformance {{marker}}?' },
requiresSynthesizeFlag: true,
// Either a schema-valid answer (key configured) or a clean unavailable
// error (no key). The runner accepts both; anything else fails.
// v0.45.x: the answer path also validates the additive compose-status
// fields (synthesis_status enum incl. extractive_fallback, pages_gathered,
// takes_gathered, warnings) via RESPONSE_SCHEMAS.synthesize — present on
// post-v0.45.x servers, never required (additive-forever).
},
{
name: 'synthesize without an LLM is a clean unavailable error (never a raw throw) — cost-gated, pass --synthesize',
verb: 'synthesize',
params: { question: 'Conformance {{marker}}: keyless error-path probe' },
requiresSynthesizeFlag: true,
// The ERROR-path fixture for synthesize. Dual-mode by necessity (a keyed
// external server legitimately answers, and the runner's synthesize
// branch accepts that arm): what this case pins is the KEYLESS arm — a
// server with no configured LLM MUST convert that state into the protocol
// envelope `error: "unavailable"` with a non-empty suggestion and
// protocol_version 1 (validated via ERROR_SCHEMA), never a raw throw or a
// fabricated answer. CI self-certification runs the suite keyless with
// the flag ON (memory-verbs-conformance.test.ts), so the error arm is
// exactly what CI asserts.
},
// ── v0.45.7 additive verbs: context_pack + delta ──────────────────────────
// The runner SKIPS these against endpoints that don't advertise the verbs
// (they are additive — a pre-v0.45.7 v1 endpoint must still certify).
{
name: 'context_pack returns a schema-valid bundle for unknown entities (empty, not an error)',
verb: 'context_pack',
params: { entities: 'conformance-nonexistent-{{marker}}', budget_tokens: 500 },
validateSchema: true,
expect: [
{ path: 'protocol_version', equals: 1 },
],
},
{
name: 'context_pack with a malformed since is invalid_params with a suggestion',
verb: 'context_pack',
params: { entities: 'conformance-nonexistent-{{marker}}', since: 'not-a-timestamp' },
expectErrorCode: 'invalid_params',
expectSuggestion: true,
},
{
name: 'delta with an explicit epoch since returns a schema-valid delta',
verb: 'delta',
params: { since: '1970-01-01T00:00:00Z', budget_tokens: 500 },
validateSchema: true,
expect: [
{ path: 'protocol_version', equals: 1 },
// `since` is normalized to ISO (v0.45.7 F4 — never echoed raw).
{ path: 'since', equals: '1970-01-01T00:00:00.000Z' },
],
},
{
name: 'delta without since or session_id is invalid_params with a suggestion',
verb: 'delta',
params: {},
expectErrorCode: 'invalid_params',
expectSuggestion: true,
},
];