Skip to content

armgit / ruflo

Public Imported from GitHub
0

🌊 The original agent harness. Deploy intelligent multi-player swarms, coordinate autonomous workflows, and build conversational AI systems. Features adaptive memory, self-learning intelligence, federation, vector RAG integration, and native Claude Code / Codex / Hermes and many more Integrated

Merge main into fix/rvf-truncated-tail-recovery; refuse oversized-prefix tails and oversized records

ree2f2990256b
diff --cc tests/rvf-tail-recovery-matrix.test.mjs
index 0000000000,0000000000..ebb2dae6d6
new file mode 100644
--- /dev/null
+++ b/tests/rvf-tail-recovery-matrix.test.mjs
@@@ -1,0 -1,0 +1,98 @@@
++import { test } from 'vitest';
++import assert from 'node:assert/strict';
++import { mkdtempSync, appendFileSync, rmSync, readFileSync, statSync } from 'node:fs';
++import { tmpdir } from 'node:os';
++import { join } from 'node:path';
++import { RvfEventLog } from '../v3/@claude-flow/shared/src/events/rvf-event-log.ts';
++
++const event = id => ({ id, aggregateId: 'a', aggregateType: 'Task', type: 'Created', timestamp: Number(id), version: 0, payload: {} });
++const ids = async log => (await log.getEvents('a')).map(e => e.id);
++
++async function withLog(fn, config = {}) {
++ const dir = mkdtempSync(join(tmpdir(), 'rvf-matrix-'));
++ const path = join(dir, 'events.rvf');
++ try { await fn(path, () => new RvfEventLog({ logPath: path, ...config })); } finally { rmSync(dir, { recursive: true, force: true }); }
++}
++
++// Repairable tails: every shape that is a torn write must lose nothing but the torn bytes, across repeated reopen→append cycles.
++const repairable = {
++ 'one byte of a length prefix': Buffer.from([0]),
++ 'a partial length prefix': Buffer.from([0, 0]),
++ 'three bytes of a length prefix': Buffer.from([0, 0, 0]),
++ 'a length prefix with no payload': Buffer.from([0, 0, 0, 100]),
++ 'a partial payload': Buffer.from([0, 0, 0, 100, 123]),
++};
++for (const [name, tail] of Object.entries(repairable)) {
++ test(`keeps every append across two reopen cycles after ${name}`, () => withLog(async (path, open) => {
++ let log = open();
++ await log.initialize(); await log.append(event('1')); await log.close();
++ appendFileSync(path, tail);
++ for (const id of ['2', '3']) { log = open(); await log.initialize(); await log.append(event(id)); await log.close(); }
++ log = open(); await log.initialize();
++ assert.deepEqual(await ids(log), ['1', '2', '3']);
++ await log.close();
++ }));
++}
++
++test('an oversized length prefix is never accepted silently: append rejects and the bytes are kept', () => withLog(async (path, open) => {
++ let log = open();
++ await log.initialize(); await log.append(event('1')); await log.close();
++ appendFileSync(path, Buffer.from('garbage')); // prefix decodes to ~1.7 GB, over the limit
++ const before = readFileSync(path);
++ log = open(); await log.initialize();
++ assert.deepEqual(await ids(log), ['1']);
++ await assert.rejects(log.append(event('2')), /unrepaired tail.*safety limit/i);
++ assert.deepEqual(readFileSync(path), before, 'file untouched');
++ assert.deepEqual(await ids(log), ['1'], 'rejected event is not indexed');
++ await log.close();
++}));
++
++test('repairOversizedTail truncates at the bad prefix so later appends are readable', () => withLog(async (path, open) => {
++ let log = open();
++ await log.initialize(); await log.append(event('1')); await log.close();
++ const good = statSync(path).size;
++ appendFileSync(path, Buffer.from('garbage'));
++ for (const id of ['2', '3']) { log = open(); await log.initialize(); await log.append(event(id)); await log.close(); }
++ log = open(); await log.initialize();
++ assert.deepEqual(await ids(log), ['1', '2', '3']);
++ assert.ok(statSync(path).size > good);
++ await log.close();
++}, { repairOversizedTail: true }));
++
++test('a complete record with a non-JSON payload and a zero-filled tail stay readable and appendable', () => withLog(async (path, open) => {
++ let log = open();
++ await log.initialize(); await log.append(event('1')); await log.close();
++ appendFileSync(path, Buffer.from([0, 0, 0, 3, 0x61, 0x62, 0x63])); // complete record, not JSON: skipped on replay
++ log = open(); await log.initialize(); await log.append(event('2')); await log.close();
++ log = open(); await log.initialize();
++ assert.deepEqual(await ids(log), ['1', '2']);
++ await log.close();
++}));
++
++test('a damaged snapshot file tail does not lose later snapshots', () => withLog(async (path, open) => {
++ let log = open();
++ await log.initialize(); await log.append(event('1'));
++ await log.saveSnapshot({ aggregateId: 'a', version: 1, state: { n: 1 }, timestamp: 1 });
++ await log.close();
++ appendFileSync(path.replace(/\.rvf$/, '.snap.rvf'), Buffer.from([0, 0]));
++ log = open(); await log.initialize();
++ await log.saveSnapshot({ aggregateId: 'a', version: 2, state: { n: 2 }, timestamp: 2 });
++ await log.close();
++ log = open(); await log.initialize();
++ assert.equal((await log.getSnapshot('a')).version, 2);
++ await log.close();
++}));
++
++test('a record larger than the replay limit is refused at append instead of being written unreadable', () => withLog(async (path, open) => {
++ const log = open();
++ await log.initialize(); await log.append(event('1'));
++ const before = readFileSync(path);
++ const huge = { ...event('2'), payload: { blob: 'x'.repeat(101 * 1024 * 1024) } };
++ await assert.rejects(log.append(huge), /exceeds the \d+ byte limit/);
++ assert.deepEqual(readFileSync(path), before);
++ await log.append(event('3'));
++ await log.close();
++ const again = open(); await again.initialize();
++ assert.deepEqual(await ids(again), ['1', '3']);
++ await again.close();
++}));
diff --cc v3/@claude-flow/shared/src/events/rvf-event-log.ts
index 73ff56fa55,690081e7d8..270964cbae
--- a/v3/@claude-flow/shared/src/events/rvf-event-log.ts
+++ b/v3/@claude-flow/shared/src/events/rvf-event-log.ts
@@@ -39,12 -39,12 +39,19 @@@ export interface RvfEventLogConfig
verbose?: boolean;
/** Maximum events before snapshot recommendation */
snapshotThreshold?: number;
++ /**
++ * When replay stops at a length prefix larger than the safety limit, truncate the file at that
++ * prefix (discarding it and everything after it) instead of refusing further appends. Default
++ * false: the bytes are kept and append() rejects explicitly until the file is repaired.
++ */
++ repairOversizedTail?: boolean;
}
const DEFAULT_CONFIG: Required<RvfEventLogConfig> = {
logPath: 'events.rvf',
verbose: false,
snapshotThreshold: 100,
++ repairOversizedTail: false,
};
// =============================================================================
@@@ -55,6 -55,6 +62,8 @@@
const MAGIC = Buffer.from('RVFL');
const MAGIC_LENGTH = 4;
const LENGTH_PREFIX_BYTES = 4;
++/** Largest record replayFile() will read; appendRecord() refuses anything bigger so no accepted append is unreadable. */
++const MAX_PAYLOAD_SIZE = 100 * 1024 * 1024; // 100MB safety limit
// =============================================================================
// RvfEventLog Implementation
@@@ -64,9 -64,6 +73,10 @@@ export class RvfEventLog extends EventE
private config: Required<RvfEventLogConfig>;
private initialized = false;
+ /** Files whose incomplete tails could not be repaired because they are read-only. */
- private readOnlyFiles = new Set<string>();
++ /** Files that hold bytes we could not repair; appends to them are refused (value: why). */
++ private readOnlyFiles = new Map<string, string>();
+
/**
* All events kept in insertion order.
* Rebuilt from the file on initialize().
@@@ -346,15 -349,13 +363,14 @@@
}
let offset = MAGIC_LENGTH;
- let recoverableTail = true;
++ let oversizedAt = -1;
-- const MAX_PAYLOAD_SIZE = 100 * 1024 * 1024; // 100MB safety limit
while (offset + LENGTH_PREFIX_BYTES <= buf.length) {
const payloadLength = buf.readUInt32BE(offset);
offset += LENGTH_PREFIX_BYTES;
if (payloadLength > MAX_PAYLOAD_SIZE) {
- recoverableTail = false;
++ oversizedAt = offset - LENGTH_PREFIX_BYTES;
if (this.config.verbose) {
console.warn(`[RvfEventLog] Payload size ${payloadLength} exceeds safety limit`);
}
@@@ -385,29 -385,13 +401,45 @@@
}
}
}
+
- if (recoverableTail && offset < buf.length) {
++ // An oversized length prefix is not a torn write: keep the bytes unless the caller opted in to repair.
++ let keepUntil = offset;
++ if (oversizedAt >= 0) {
++ if (!this.config.repairOversizedTail) {
++ this.readOnlyFiles.set(filePath, 'a record length prefix exceeds the safety limit');
++ return;
++ }
++ keepUntil = oversizedAt;
++ }
++
++ if (keepUntil < buf.length) {
+ try {
- truncateSync(filePath, offset);
++ truncateSync(filePath, keepUntil);
+ } catch (error) {
+ const code = (error as NodeJS.ErrnoException).code;
+ if (code !== 'EACCES' && code !== 'EROFS' && code !== 'EPERM') throw error;
+ // Preserve readable history but never append behind an unreadable tail.
- this.readOnlyFiles.add(filePath);
++ this.readOnlyFiles.set(filePath, 'the file is not writable, so its damaged tail could not be repaired');
+ if (this.config.verbose) {
+ console.warn(`[RvfEventLog] Read-only log has an unrepaired tail: ${filePath}`);
+ }
+ }
+ }
}
/** Append a single record to an RVF file. */
- private appendRecord(filePath: string, record: unknown): void {
- if (this.readOnlyFiles.has(filePath)) {
- throw new Error(`[RvfEventLog] Read-only log has an unrepaired tail; append refused: ${filePath}`);
+ private appendRecord<T>(filePath: string, record: T): T {
++ const refused = this.readOnlyFiles.get(filePath);
++ if (refused !== undefined) {
++ throw new Error(`[RvfEventLog] Read-only until repaired: unrepaired tail (${refused}); append refused: ${filePath}`);
+ }
const json = JSON.stringify(record);
+ const persisted = JSON.parse(json) as T;
const payload = Buffer.from(json, 'utf8');
++ if (payload.length > MAX_PAYLOAD_SIZE) {
++ // replayFile() refuses records this large, so writing one would be an append that can never be read back.
++ throw new Error(`[RvfEventLog] Record of ${payload.length} bytes exceeds the ${MAX_PAYLOAD_SIZE} byte limit; append refused`);
++ }
const lengthBuf = Buffer.allocUnsafe(LENGTH_PREFIX_BYTES);
lengthBuf.writeUInt32BE(payload.length, 0);