Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(plugins): drain the dsh pending queue in-process so a transient write failure self-heals (#4779)
Nick James3841e6f2992b
diff --git a/examples/claude-code-memory-plugin/scripts/shared/pending-queue.mjs b/examples/claude-code-memory-plugin/scripts/shared/pending-queue.mjsindex 4b3836a4b..38dccb1c8 100644--- a/examples/claude-code-memory-plugin/scripts/shared/pending-queue.mjs+++ b/examples/claude-code-memory-plugin/scripts/shared/pending-queue.mjs@@ -4,9 +4,10 @@** When the OpenViking server is temporarily unreachable, write operations* (addMessage, commitSession) serialize their payloads to- * `~/.openviking/pending/` as JSON files. On the next session-start, the- * queue is replayed in small batches. This is a session-start-triggered retry- * path with maxRetries/TTL, not a long-running background worker.+ * `~/.openviking/pending/` as JSON files. The queue is replayed in small+ * batches, either at session-start (consuming retry budgets, with+ * maxRetries/TTL) or by a long-running drainer that passes+ * `consumeRetries: false` so transient failures stay retryable.** Each file contains: { type, sessionId, payload, createdAt, retries, dedupKey }*@@ -274,6 +275,23 @@ export async function claimForReplay(filename) {}}+/**+ * Release a claimed file back to the queue without consuming a retry. The+ * entry keeps its original filename, retry count, and createdAt position, so a+ * later run retries it as if this attempt never happened.+ */+export async function releaseClaim(claimedFilename) {+ if (!claimedFilename.endsWith(".processing")) return null;+ const dir = getPendingDir();+ const restored = pendingFromProcessingFilename(claimedFilename);+ try {+ await rename(join(dir, claimedFilename), join(dir, restored));+ return restored;+ } catch {+ return null;+ }+}+/*** Remove a pending entry after successful replay.*/@@ -350,9 +368,16 @@ export async function cleanStale() {** @param {Function} fetchJSON - the configured fetchJSON from makeFetchJSON* @param {Function} log - logger function+ * @param {object} [options]+ * @param {boolean} [options.consumeRetries=true] - when false, retryable+ * failures release their claim instead of incrementing the retry count, so+ * a background drainer can keep retrying a transient failure without+ * burning the session-start retry budget. Exhausted and non-retryable+ * entries are deleted exactly as in the default mode.* @returns {{ replayed: number, failed: number, skipped: number, deferred: number }}*/-export async function replayPending(fetchJSON, log) {+export async function replayPending(fetchJSON, log, options = {}) {+ const consumeRetries = options.consumeRetries !== false;const pending = await listPending();if (pending.length === 0) {@@ -427,7 +452,18 @@ export async function replayPending(fetchJSON, log) {await dequeue(claimedFilename);skipped++;} else {- await incrementRetry(claimedFilename, entry);+ if (consumeRetries) {+ await incrementRetry(claimedFilename, entry);+ } else {+ const released = await releaseClaim(claimedFilename);+ log("pending-queue", {+ action: released ? "replay-deferred" : "release-failed",+ sessionId: entry.sessionId,+ type: entry.type,+ status: res?.result?.status || res?.status,+ retries: entry.retries || 0,+ });+ }failed++;if (entry.type === "addMessage") {deferred += Math.max(0, pending.length - processed);diff --git a/examples/codex-memory-plugin/scripts/shared/pending-queue.mjs b/examples/codex-memory-plugin/scripts/shared/pending-queue.mjsindex 4b3836a4b..38dccb1c8 100644--- a/examples/codex-memory-plugin/scripts/shared/pending-queue.mjs+++ b/examples/codex-memory-plugin/scripts/shared/pending-queue.mjs@@ -4,9 +4,10 @@** When the OpenViking server is temporarily unreachable, write operations* (addMessage, commitSession) serialize their payloads to- * `~/.openviking/pending/` as JSON files. On the next session-start, the- * queue is replayed in small batches. This is a session-start-triggered retry- * path with maxRetries/TTL, not a long-running background worker.+ * `~/.openviking/pending/` as JSON files. The queue is replayed in small+ * batches, either at session-start (consuming retry budgets, with+ * maxRetries/TTL) or by a long-running drainer that passes+ * `consumeRetries: false` so transient failures stay retryable.** Each file contains: { type, sessionId, payload, createdAt, retries, dedupKey }*@@ -274,6 +275,23 @@ export async function claimForReplay(filename) {}}+/**+ * Release a claimed file back to the queue without consuming a retry. The+ * entry keeps its original filename, retry count, and createdAt position, so a+ * later run retries it as if this attempt never happened.+ */+export async function releaseClaim(claimedFilename) {+ if (!claimedFilename.endsWith(".processing")) return null;+ const dir = getPendingDir();+ const restored = pendingFromProcessingFilename(claimedFilename);+ try {+ await rename(join(dir, claimedFilename), join(dir, restored));+ return restored;+ } catch {+ return null;+ }+}+/*** Remove a pending entry after successful replay.*/@@ -350,9 +368,16 @@ export async function cleanStale() {** @param {Function} fetchJSON - the configured fetchJSON from makeFetchJSON* @param {Function} log - logger function+ * @param {object} [options]+ * @param {boolean} [options.consumeRetries=true] - when false, retryable+ * failures release their claim instead of incrementing the retry count, so+ * a background drainer can keep retrying a transient failure without+ * burning the session-start retry budget. Exhausted and non-retryable+ * entries are deleted exactly as in the default mode.* @returns {{ replayed: number, failed: number, skipped: number, deferred: number }}*/-export async function replayPending(fetchJSON, log) {+export async function replayPending(fetchJSON, log, options = {}) {+ const consumeRetries = options.consumeRetries !== false;const pending = await listPending();if (pending.length === 0) {@@ -427,7 +452,18 @@ export async function replayPending(fetchJSON, log) {await dequeue(claimedFilename);skipped++;} else {- await incrementRetry(claimedFilename, entry);+ if (consumeRetries) {+ await incrementRetry(claimedFilename, entry);+ } else {+ const released = await releaseClaim(claimedFilename);+ log("pending-queue", {+ action: released ? "replay-deferred" : "release-failed",+ sessionId: entry.sessionId,+ type: entry.type,+ status: res?.result?.status || res?.status,+ retries: entry.retries || 0,+ });+ }failed++;if (entry.type === "addMessage") {deferred += Math.max(0, pending.length - processed);diff --git a/examples/dsh-memory-plugin/README.md b/examples/dsh-memory-plugin/README.mdindex 08b560766..0d9a8cb09 100644--- a/examples/dsh-memory-plugin/README.md+++ b/examples/dsh-memory-plugin/README.md@@ -150,8 +150,8 @@ The patch can also carry plugin config:- `session/event` captures user, assistant, and optionally tool-result messages without scraping a transcript.- `turn/end` checks the OpenViking pending-token threshold and commits when required.- `skipSubagentSessions: true` excludes sessions marked with `header.origin: subagent` from automatic profile, recall, capture, and commit; it defaults to `false`.-- `syncTurns: false` stops every write: no captured messages, no threshold or shutdown commit, and no replay of writes an earlier session queued — those stay queued for a session that still writes. Profile injection and recall are unaffected; it defaults to `true`.-- Failed writes enter the shared OpenViking pending queue for replay at the next session start.+- `syncTurns: false` stops every new write: no captured messages, no threshold or shutdown commit. Writes queued while the toggle was on are still replayed by the background drainer once the server recovers — they were captured with the toggle on. Profile injection and recall are unaffected; it defaults to `true`.+- Failed writes enter the shared OpenViking pending queue. A background drainer (default every 60s, `OPENVIKING_PENDING_DRAIN_INTERVAL_MS`) probes the server health and replays the queue in-process, so a transient write failure recovers without restarting dsh; it does not consume the session-start retry budget. Session-start replays keep consuming retries as before.- `tools/pre-execute` blocks DSH filesystem and shell tools from treating `viking://` URIs as local paths, pointing the model at the bridged `mcp__openviking__*` tools instead.Each DSH session maps to `dsh-<session-id>` in OpenViking. Workspace-derived actor peers are resolved per session and sent on every session-specific request: the peer is the git identity of the session's workspace — the normalized `origin` URL (`[email protected]:volcengine/OpenViking.git` becomes `github.com-volcengine-openviking`), else the repository root path, that fallback keeping the older rule where every non-letter-or-digit character becomes `-`. Outside a git repository no peer is sent at all, and what is remembered there goes to the user-level space `viking://user/<you>/memories`. One repository therefore keeps one peer across subdirectories, worktrees, clones and machines, while a fork's different origin keeps it separate. DSH does not read workspace `.openviking/config.json` files, so a `peer.id` written there has no effect; pin a peer with `OPENVIKING_PEER_ID` instead. Memories written under the older path-derived peer stay reachable: the default `recallPeerScope: all` sweeps every peer under the user.diff --git a/examples/dsh-memory-plugin/index.mjs b/examples/dsh-memory-plugin/index.mjsindex 69d6623cd..f6a738efd 100644--- a/examples/dsh-memory-plugin/index.mjs+++ b/examples/dsh-memory-plugin/index.mjs@@ -21,6 +21,14 @@ export function apply(ctx, input = {}) {() => () => runtime.disposeAll(),"openvikingMemory.disposeAll()",);+ // The pending-queue drainer is the in-process recovery path: without it a+ // single transient write failure latches capture/commit until the next dsh+ // restart. Started here so every session shares one single-flight drainer.+ runtime.startDrainer();+ ctx.effect(+ () => () => runtime.stopDrainer(),+ "openvikingMemory.stopDrainer()",+ );ctx.on("agent/session-start", ({ agent }) => {if (skipMemory(agent.session)) return false;diff --git a/examples/dsh-memory-plugin/pending-queue.test.mjs b/examples/dsh-memory-plugin/pending-queue.test.mjsnew file mode 100644index 000000000..1173a6a18--- /dev/null+++ b/examples/dsh-memory-plugin/pending-queue.test.mjs@@ -0,0 +1,180 @@+import assert from "node:assert/strict";+import { mkdtemp, rm } from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";+import test from "node:test";++import {+ enqueue,+ listPending,+ replayPending,+} from "./shared/pending-queue.mjs";++const originalEnv = {+ dir: process.env.OPENVIKING_PENDING_DIR,+ maxRetries: process.env.OPENVIKING_PENDING_MAX_RETRIES,+ replayLimit: process.env.OPENVIKING_PENDING_REPLAY_LIMIT,+ ttlDays: process.env.OPENVIKING_PENDING_TTL_DAYS,+};++async function withPendingDir(fn) {+ const dir = await mkdtemp(join(tmpdir(), "dsh-pending-test-"));+ process.env.OPENVIKING_PENDING_DIR = dir;+ delete process.env.OPENVIKING_PENDING_MAX_RETRIES;+ delete process.env.OPENVIKING_PENDING_REPLAY_LIMIT;+ delete process.env.OPENVIKING_PENDING_TTL_DAYS;+ try {+ return await fn(dir);+ } finally {+ if (originalEnv.dir === undefined) delete process.env.OPENVIKING_PENDING_DIR;+ else process.env.OPENVIKING_PENDING_DIR = originalEnv.dir;+ if (originalEnv.maxRetries === undefined) delete process.env.OPENVIKING_PENDING_MAX_RETRIES;+ else process.env.OPENVIKING_PENDING_MAX_RETRIES = originalEnv.maxRetries;+ if (originalEnv.replayLimit === undefined) delete process.env.OPENVIKING_PENDING_REPLAY_LIMIT;+ else process.env.OPENVIKING_PENDING_REPLAY_LIMIT = originalEnv.replayLimit;+ if (originalEnv.ttlDays === undefined) delete process.env.OPENVIKING_PENDING_TTL_DAYS;+ else process.env.OPENVIKING_PENDING_TTL_DAYS = originalEnv.ttlDays;+ await rm(dir, { recursive: true, force: true });+ }+}++test("replay with consumeRetries:false keeps a retryable failure intact for the next attempt", async () => {+ await withPendingDir(async () => {+ const payload = { role: "user", content: "remember this" };+ await enqueue("addMessage", "dsh-drain", payload);++ let calls = 0;+ const first = await replayPending(+ async () => {+ calls += 1;+ return { ok: false, status: 503, error: { message: "unavailable" } };+ },+ () => {},+ { consumeRetries: false },+ );++ assert.equal(first.replayed, 0);+ assert.equal(first.failed, 1);+ assert.equal(calls, 1);++ const pending = await listPending();+ assert.equal(pending.length, 1);+ assert.equal(pending[0].entry.retries, 0);+ assert.ok(pending[0].filename.endsWith("_0.json"), pending[0].filename);+ assert.deepEqual(pending[0].entry.payload, payload);++ const recovered = await replayPending(+ async () => ({ ok: true }),+ () => {},+ { consumeRetries: false },+ );+ assert.equal(recovered.replayed, 1);+ assert.deepEqual(await listPending(), []);+ });+});++test("replay with consumeRetries:false still deletes exhausted entries", async () => {+ await withPendingDir(async () => {+ process.env.OPENVIKING_PENDING_MAX_RETRIES = "0";+ await enqueue("addMessage", "dsh-exhausted", { content: "doomed" });++ const result = await replayPending(+ async () => ({ ok: true }),+ () => {},+ { consumeRetries: false },+ );++ assert.equal(result.replayed, 0);+ assert.equal(result.skipped, 1);+ assert.deepEqual(await listPending(), []);+ });+});++test("replay with consumeRetries:false still deletes non-retryable failures", async () => {+ await withPendingDir(async () => {+ await enqueue("addMessage", "dsh-rejected", { content: "too large" });++ const result = await replayPending(+ async () => ({ ok: false, status: 413, error: { message: "too large" } }),+ () => {},+ { consumeRetries: false },+ );++ assert.equal(result.replayed, 0);+ assert.equal(result.skipped, 1);+ assert.deepEqual(await listPending(), []);+ });+});++test("replay with consumeRetries:false stops at the first retryable failure and preserves order", async () => {+ await withPendingDir(async () => {+ await enqueue("addMessage", "dsh-order", { content: "first" });+ await enqueue("addMessage", "dsh-order", { content: "second" });++ let calls = 0;+ const result = await replayPending(+ async () => {+ calls += 1;+ return { ok: false, status: 503, error: { message: "unavailable" } };+ },+ () => {},+ { consumeRetries: false },+ );++ assert.equal(calls, 1);+ assert.equal(result.failed, 1);+ const pending = await listPending();+ assert.deepEqual(+ pending.map(item => item.entry.payload.content),+ ["first", "second"],+ );+ assert.equal(pending[0].entry.retries, 0);+ });+});++test("replay with consumeRetries:false keeps going after a failed commitSession", async () => {+ await withPendingDir(async () => {+ await enqueue("commitSession", "dsh-commit", { keep_recent_count: 10 });+ await enqueue("addMessage", "dsh-commit", { content: "after the commit" });++ let calls = 0;+ const result = await replayPending(+ async (_path, init) => {+ calls += 1;+ if (init?.body && init.body.includes("keep_recent_count")) {+ return { ok: false, status: 503, error: { message: "unavailable" } };+ }+ return { ok: true };+ },+ () => {},+ { consumeRetries: false },+ );++ assert.equal(calls, 2);+ assert.equal(result.failed, 1);+ assert.equal(result.replayed, 1);+ const pending = await listPending();+ assert.deepEqual(+ pending.map(item => item.entry.type),+ ["commitSession"],+ );+ assert.equal(pending[0].entry.retries, 0);+ });+});++test("replay still consumes retries by default", async () => {+ await withPendingDir(async () => {+ await enqueue("addMessage", "dsh-default", { content: "flaky" });++ const result = await replayPending(+ async () => ({ ok: false, status: 503, error: { message: "unavailable" } }),+ () => {},+ );++ assert.equal(result.failed, 1);+ const pending = await listPending();+ assert.equal(pending.length, 1);+ assert.equal(pending[0].entry.retries, 1);+ assert.ok(pending[0].filename.endsWith("_1.json"), pending[0].filename);+ });+});diff --git a/examples/dsh-memory-plugin/runtime-drain.test.mjs b/examples/dsh-memory-plugin/runtime-drain.test.mjsnew file mode 100644index 000000000..f1d9a0e55--- /dev/null+++ b/examples/dsh-memory-plugin/runtime-drain.test.mjs@@ -0,0 +1,335 @@+import assert from "node:assert/strict";+import { mkdtemp, rm } from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { afterEach, test } from "node:test";+import { enqueue, listPending } from "./shared/pending-queue.mjs";+import { OpenVikingRuntime } from "./runtime.mjs";++const originalPendingDir = process.env.OPENVIKING_PENDING_DIR;+const originalDrainInterval = process.env.OPENVIKING_PENDING_DRAIN_INTERVAL_MS;+const tempDirs = [];++afterEach(async () => {+ if (originalPendingDir === undefined) delete process.env.OPENVIKING_PENDING_DIR;+ else process.env.OPENVIKING_PENDING_DIR = originalPendingDir;+ if (originalDrainInterval === undefined) delete process.env.OPENVIKING_PENDING_DRAIN_INTERVAL_MS;+ else process.env.OPENVIKING_PENDING_DRAIN_INTERVAL_MS = originalDrainInterval;+ await Promise.all(tempDirs.splice(0).map(dir => rm(dir, { recursive: true, force: true })));+});++test("drainTick sends queued messages once the server recovers and clears the latch", async () => {+ const pendingDir = await mkdtemp(join(tmpdir(), "dsh-drain-"));+ tempDirs.push(pendingDir);+ process.env.OPENVIKING_PENDING_DIR = pendingDir;++ let healthy = false;+ let addCalls = 0;+ const runtime = new OpenVikingRuntime({+ async addMessage() {+ addCalls += 1;+ return healthy+ ? { ok: true }+ : { ok: false, status: 503, error: { message: "down" } };+ },+ async fetchJSON() {+ return healthy+ ? { ok: true }+ : { ok: false, status: 503, error: { message: "down" } };+ },+ async healthResult() {+ return healthy+ ? { ok: true }+ : { ok: false, status: 503, error: { message: "down" } };+ },+ }, config(), { debug() {} });+ const session = { id: "recover", header: { cwd: "/workspace" } };+ runtime.stateFor(session).ready = true;++ runtime.capture(session, userEvent("Queued during the outage."));+ await runtime.flush(session);+ assert.equal((await listPending()).length, 1);+ assert.equal(runtime.stateFor(session).hasPendingWrites, true);++ healthy = true;+ await runtime.drainTick();++ assert.equal((await listPending()).length, 0);+ assert.equal(runtime.stateFor(session).hasPendingWrites, false);++ runtime.capture(session, userEvent("Sent directly after recovery."));+ await runtime.flush(session);+ assert.equal(addCalls, 2);+ assert.equal((await listPending()).length, 0);+});++test("drainTick keeps the latch while the server stays down and leaves the entry retryable", async () => {+ const pendingDir = await mkdtemp(join(tmpdir(), "dsh-drain-down-"));+ tempDirs.push(pendingDir);+ process.env.OPENVIKING_PENDING_DIR = pendingDir;++ let addCalls = 0;+ const runtime = new OpenVikingRuntime({+ async addMessage() {+ addCalls += 1;+ return { ok: false, status: 503, error: { message: "down" } };+ },+ async fetchJSON() {+ return { ok: false, status: 503, error: { message: "down" } };+ },+ async healthResult() {+ return { ok: false, status: 503, error: { message: "down" } };+ },+ }, config(), { debug() {} });+ const session = { id: "still-down", header: { cwd: "/workspace" } };+ runtime.stateFor(session).ready = true;++ runtime.capture(session, userEvent("First queued message."));+ await runtime.flush(session);+ await runtime.drainTick();++ assert.equal(runtime.stateFor(session).hasPendingWrites, true);+ const pending = await listPending();+ assert.equal(pending.length, 1);+ assert.equal(pending[0].entry.retries, 0);++ runtime.capture(session, userEvent("Second queued message."));+ await runtime.flush(session);+ assert.equal(addCalls, 1);+ assert.equal((await listPending()).length, 2);+});++test("drainTick does not hit the network while the queue is empty", async () => {+ let fetchCalls = 0;+ const runtime = new OpenVikingRuntime({+ async fetchJSON() {+ fetchCalls += 1;+ return { ok: true };+ },+ }, config(), { debug() {} });+ runtime.stateFor({ id: "idle", header: { cwd: "/workspace" } });++ await runtime.drainTick();++ assert.equal(fetchCalls, 0);+});++test("commit resumes once the drain clears the backlog", async () => {+ const pendingDir = await mkdtemp(join(tmpdir(), "dsh-commit-resume-"));+ tempDirs.push(pendingDir);+ process.env.OPENVIKING_PENDING_DIR = pendingDir;++ let healthy = false;+ let commitCalls = 0;+ const runtime = new OpenVikingRuntime({+ async addMessage() {+ return healthy+ ? { ok: true }+ : { ok: false, status: 503, error: { message: "down" } };+ },+ async fetchJSON() {+ return healthy+ ? { ok: true }+ : { ok: false, status: 503, error: { message: "down" } };+ },+ async healthResult() {+ return healthy+ ? { ok: true }+ : { ok: false, status: 503, error: { message: "down" } };+ },+ async getSession() {+ return { pending_tokens: 20000 };+ },+ async commitSession() {+ commitCalls += 1;+ return { ok: true };+ },+ }, config(), { debug() {} });+ const session = { id: "commit-resume", header: { cwd: "/workspace" } };+ const state = runtime.stateFor(session);+ state.ready = true;++ runtime.capture(session, userEvent("Queued during the outage."));+ await runtime.flush(session);+ assert.equal(state.hasPendingWrites, true);++ // While the backlog holds, the commit check keeps its ordering guard.+ runtime.maybeCommit(session, { type: "turn/end" });+ await runtime.flush(session);+ assert.equal(commitCalls, 0);++ // The drainer clears the backlog, and the next turn/end commits again.+ healthy = true;+ await runtime.drainTick();+ assert.equal(state.hasPendingWrites, false);++ runtime.maybeCommit(session, { type: "turn/end" });+ await runtime.flush(session);+ assert.equal(commitCalls, 1);+});++test("startDrainer drains the queue on its interval", async t => {+ t.mock.timers.enable({ apis: ["setInterval"] });+ const pendingDir = await mkdtemp(join(tmpdir(), "dsh-drain-interval-"));+ tempDirs.push(pendingDir);+ process.env.OPENVIKING_PENDING_DIR = pendingDir;+ process.env.OPENVIKING_PENDING_DRAIN_INTERVAL_MS = "60000";++ let healthy = false;+ const runtime = new OpenVikingRuntime({+ async addMessage() {+ return healthy+ ? { ok: true }+ : { ok: false, status: 503, error: { message: "down" } };+ },+ async fetchJSON() {+ return healthy+ ? { ok: true }+ : { ok: false, status: 503, error: { message: "down" } };+ },+ async healthResult() {+ return healthy+ ? { ok: true }+ : { ok: false, status: 503, error: { message: "down" } };+ },+ }, config(), { debug() {} });+ const session = { id: "interval", header: { cwd: "/workspace" } };+ runtime.stateFor(session).ready = true;++ runtime.capture(session, userEvent("Queued."));+ await runtime.flush(session);+ assert.equal((await listPending()).length, 1);++ runtime.startDrainer();+ healthy = true;+ t.mock.timers.tick(60000);+ await runtime.drainPromise;++ assert.equal((await listPending()).length, 0);+ assert.equal(runtime.stateFor(session).hasPendingWrites, false);++ runtime.stopDrainer();+});++test("drainTick runs single-flight", async () => {+ const pendingDir = await mkdtemp(join(tmpdir(), "dsh-drain-single-"));+ tempDirs.push(pendingDir);+ process.env.OPENVIKING_PENDING_DIR = pendingDir;+ await enqueue("addMessage", "dsh-single", { content: "held" });++ let release;+ let startedResolve;+ const started = new Promise(resolve => {+ startedResolve = resolve;+ });+ let fetchCalls = 0;+ const runtime = new OpenVikingRuntime({+ async healthResult() {+ return { ok: true };+ },+ async fetchJSON() {+ fetchCalls += 1;+ startedResolve();+ await new Promise(resolve => {+ release = resolve;+ });+ return { ok: true };+ },+ }, config(), { debug() {} });+ runtime.stateFor({ id: "single", header: { cwd: "/workspace" } }).ready = true;++ const first = runtime.drainTick();+ await started;+ await runtime.drainTick();+ release();+ await first;++ assert.equal(fetchCalls, 1);+});++test("drainTick derives each session's latch from its own entries", async () => {+ const pendingDir = await mkdtemp(join(tmpdir(), "dsh-drain-isolate-"));+ tempDirs.push(pendingDir);+ process.env.OPENVIKING_PENDING_DIR = pendingDir;++ const runtime = new OpenVikingRuntime({+ async addMessage() {+ return { ok: false, status: 503, error: { message: "down" } };+ },+ async healthResult() {+ return { ok: true };+ },+ async fetchJSON() {+ return { ok: false, status: 503, error: { message: "down" } };+ },+ }, config(), { debug() {} });+ const backlogged = { id: "backlogged", header: { cwd: "/workspace" } };+ const clean = { id: "clean", header: { cwd: "/workspace" } };+ runtime.stateFor(backlogged).ready = true;+ runtime.stateFor(clean).ready = true;++ runtime.capture(backlogged, userEvent("Queued."));+ await runtime.flush(backlogged);+ await runtime.drainTick();++ assert.equal(runtime.stateFor(backlogged).hasPendingWrites, true);+ assert.equal(runtime.stateFor(clean).hasPendingWrites, false);+});++test("startDrainer is idempotent, stopDrainer is safe, and an invalid env falls back to 60s", async t => {+ t.mock.timers.enable({ apis: ["setInterval"] });+ const pendingDir = await mkdtemp(join(tmpdir(), "dsh-drain-fallback-"));+ tempDirs.push(pendingDir);+ process.env.OPENVIKING_PENDING_DIR = pendingDir;+ process.env.OPENVIKING_PENDING_DRAIN_INTERVAL_MS = "not-a-number";+ await enqueue("addMessage", "dsh-fallback", { content: "held" });++ let fetchCalls = 0;+ const runtime = new OpenVikingRuntime({+ async healthResult() {+ return { ok: true };+ },+ async fetchJSON() {+ fetchCalls += 1;+ return { ok: true };+ },+ }, config(), { debug() {} });+ runtime.stateFor({ id: "fallback", header: { cwd: "/workspace" } }).ready = true;++ const timer = runtime.startDrainer();+ assert.equal(runtime.startDrainer(), timer);+ t.mock.timers.tick(59999);+ assert.equal(fetchCalls, 0);+ t.mock.timers.tick(1);+ await runtime.drainPromise;+ assert.ok(fetchCalls >= 1);++ runtime.stopDrainer();+ runtime.stopDrainer();+});++function config() {+ return {+ explicitPeerId: "",+ workspacePeer: false,+ peerId: "",+ syncTurns: true,+ captureAssistantTurns: true,+ captureToolResults: false,+ captureToolMaxChars: 1000000,+ captureMaxLength: 24000,+ captureMode: "semantic",+ commitKeepRecentCount: 10,+ };+}++function userEvent(text) {+ return {+ type: "user/message",+ data: {+ role: "user",+ content: [{ type: "text", text }],+ source: { kind: "user" },+ },+ };+}diff --git a/examples/dsh-memory-plugin/runtime.mjs b/examples/dsh-memory-plugin/runtime.mjsindex 1dc9f8137..0449b2b67 100644--- a/examples/dsh-memory-plugin/runtime.mjs+++ b/examples/dsh-memory-plugin/runtime.mjs@@ -22,6 +22,10 @@ export class OpenVikingRuntime {this.config = config;this.logger = logger;this.states = new Map();+ this.drainTimer = null;+ this.drainRunning = false;+ this.drainPromise = Promise.resolve();+ this.drainHealth = null;}stateFor(session) {@@ -90,10 +94,7 @@ export class OpenVikingRuntime {// Replay is a write, so it stays behind the same toggle: a backlog queued// while capture was on waits for a session that still writes.if (state.config.syncTurns) {- await replayPending(- (path, init) => this.client.fetchJSON(path, init),- (stage, data) => this.log(stage, data),- );+ await this.replayPendingQueue();}await this.refreshPendingState(state);const profile = await buildProfileBlock(@@ -260,7 +261,15 @@ export class OpenVikingRuntime {const createdAt = Math.max(Date.now(), state.pendingCreatedAt + 1);state.pendingCreatedAt = createdAt;const result = await enqueue(type, state.ovSessionId, payload, { createdAt });- if (result.ok) state.hasPendingWrites = true;+ if (result.ok) {+ // Log the latch transition only: every message that follows while the+ // latch holds takes the cheap enqueue path, so this fires once per+ // outage, not once per message.+ if (!state.hasPendingWrites) {+ this.log("pending_latched", { sessionId: state.ovSessionId, type });+ }+ state.hasPendingWrites = true;+ }if (!result.ok) {this.log("pending_enqueue_error", {sessionId: state.ovSessionId,@@ -295,6 +304,7 @@ export class OpenVikingRuntime {}async refreshPendingState(state) {+ const wasPending = state.hasPendingWrites;const pending = (await listPending()).filter(item => item.entry?.sessionId === state.ovSessionId,);@@ -303,6 +313,84 @@ export class OpenVikingRuntime {(latest, item) => Math.max(latest, Number(item.entry?.createdAt || 0)),state.pendingCreatedAt,);+ // Only the flip back to direct sends is logged: the recovery moment that+ // proves the drainer worked, once per outage.+ if (wasPending && !state.hasPendingWrites) {+ this.log("pending_cleared", { sessionId: state.ovSessionId });+ }+ }++ /**+ * Replay the pending queue through this runtime's client. The session-start+ * path calls it without options and keeps consuming retries; the drainer+ * passes consumeRetries:false so transient failures stay retryable.+ */+ async replayPendingQueue(options = {}) {+ await replayPending(+ (path, init) => this.client.fetchJSON(path, init),+ (stage, data) => this.log(stage, data),+ options,+ );+ }++ /**+ * One drainer tick, following the session-start replay flow: probe health+ * first and only replay when the server answers. Replays run without+ * consuming retry budgets, then every session's latch is re-derived from+ * the queue. An empty queue clears the latches with zero HTTP traffic, so+ * once a transient write failure recovers, capture and commit resume on+ * their own without restarting the long-lived dsh process.+ */+ async drainTick() {+ if (this.drainRunning) return;+ this.drainRunning = true;+ try {+ const pending = await listPending();+ if (pending.length > 0) {+ const health = await this.client.healthResult();+ if (!health.ok) {+ // Only the flip into the outage is logged, not every 60s probe.+ if (this.drainHealth !== false) {+ this.log("drain_health_down", { status: health.status || 0 });+ }+ this.drainHealth = false;+ } else {+ if (this.drainHealth === false) {+ this.log("drain_health_restored", {});+ }+ this.drainHealth = true;+ await this.replayPendingQueue({ consumeRetries: false });+ }+ }+ for (const state of this.states.values()) {+ await this.refreshPendingState(state);+ }+ } finally {+ this.drainRunning = false;+ }+ }++ /**+ * Start the background drainer. The interval is fixed per process; each tick+ * is single-flight, so a slow replay run never overlaps the next one.+ */+ startDrainer() {+ if (this.drainTimer) return this.drainTimer;+ const parsed = parseInt(process.env.OPENVIKING_PENDING_DRAIN_INTERVAL_MS || "", 10);+ const intervalMs = Number.isFinite(parsed) && parsed > 0 ? parsed : 60000;+ this.drainTimer = setInterval(() => {+ this.drainPromise = this.drainTick().catch(error => this.log("drain_error", {+ error: error instanceof Error ? error.message : String(error),+ }));+ }, intervalMs);+ this.drainTimer.unref?.();+ return this.drainTimer;+ }++ stopDrainer() {+ if (!this.drainTimer) return;+ clearInterval(this.drainTimer);+ this.drainTimer = null;}async flush(session) {diff --git a/examples/dsh-memory-plugin/shared/pending-queue.mjs b/examples/dsh-memory-plugin/shared/pending-queue.mjsindex 4b3836a4b..38dccb1c8 100644--- a/examples/dsh-memory-plugin/shared/pending-queue.mjs+++ b/examples/dsh-memory-plugin/shared/pending-queue.mjs@@ -4,9 +4,10 @@** When the OpenViking server is temporarily unreachable, write operations* (addMessage, commitSession) serialize their payloads to- * `~/.openviking/pending/` as JSON files. On the next session-start, the- * queue is replayed in small batches. This is a session-start-triggered retry- * path with maxRetries/TTL, not a long-running background worker.+ * `~/.openviking/pending/` as JSON files. The queue is replayed in small+ * batches, either at session-start (consuming retry budgets, with+ * maxRetries/TTL) or by a long-running drainer that passes+ * `consumeRetries: false` so transient failures stay retryable.** Each file contains: { type, sessionId, payload, createdAt, retries, dedupKey }*@@ -274,6 +275,23 @@ export async function claimForReplay(filename) {}}+/**+ * Release a claimed file back to the queue without consuming a retry. The+ * entry keeps its original filename, retry count, and createdAt position, so a+ * later run retries it as if this attempt never happened.+ */+export async function releaseClaim(claimedFilename) {+ if (!claimedFilename.endsWith(".processing")) return null;+ const dir = getPendingDir();+ const restored = pendingFromProcessingFilename(claimedFilename);+ try {+ await rename(join(dir, claimedFilename), join(dir, restored));+ return restored;+ } catch {+ return null;+ }+}+/*** Remove a pending entry after successful replay.*/@@ -350,9 +368,16 @@ export async function cleanStale() {** @param {Function} fetchJSON - the configured fetchJSON from makeFetchJSON* @param {Function} log - logger function+ * @param {object} [options]+ * @param {boolean} [options.consumeRetries=true] - when false, retryable+ * failures release their claim instead of incrementing the retry count, so+ * a background drainer can keep retrying a transient failure without+ * burning the session-start retry budget. Exhausted and non-retryable+ * entries are deleted exactly as in the default mode.* @returns {{ replayed: number, failed: number, skipped: number, deferred: number }}*/-export async function replayPending(fetchJSON, log) {+export async function replayPending(fetchJSON, log, options = {}) {+ const consumeRetries = options.consumeRetries !== false;const pending = await listPending();if (pending.length === 0) {@@ -427,7 +452,18 @@ export async function replayPending(fetchJSON, log) {await dequeue(claimedFilename);skipped++;} else {- await incrementRetry(claimedFilename, entry);+ if (consumeRetries) {+ await incrementRetry(claimedFilename, entry);+ } else {+ const released = await releaseClaim(claimedFilename);+ log("pending-queue", {+ action: released ? "replay-deferred" : "release-failed",+ sessionId: entry.sessionId,+ type: entry.type,+ status: res?.result?.status || res?.status,+ retries: entry.retries || 0,+ });+ }failed++;if (entry.type === "addMessage") {deferred += Math.max(0, pending.length - processed);diff --git a/examples/memory-plugin-shared/lib/pending-queue.mjs b/examples/memory-plugin-shared/lib/pending-queue.mjsindex 6dc45b23d..0666620bc 100644--- a/examples/memory-plugin-shared/lib/pending-queue.mjs+++ b/examples/memory-plugin-shared/lib/pending-queue.mjs@@ -3,9 +3,10 @@** When the OpenViking server is temporarily unreachable, write operations* (addMessage, commitSession) serialize their payloads to- * `~/.openviking/pending/` as JSON files. On the next session-start, the- * queue is replayed in small batches. This is a session-start-triggered retry- * path with maxRetries/TTL, not a long-running background worker.+ * `~/.openviking/pending/` as JSON files. The queue is replayed in small+ * batches, either at session-start (consuming retry budgets, with+ * maxRetries/TTL) or by a long-running drainer that passes+ * `consumeRetries: false` so transient failures stay retryable.** Each file contains: { type, sessionId, payload, createdAt, retries, dedupKey }*@@ -273,6 +274,23 @@ export async function claimForReplay(filename) {}}+/**+ * Release a claimed file back to the queue without consuming a retry. The+ * entry keeps its original filename, retry count, and createdAt position, so a+ * later run retries it as if this attempt never happened.+ */+export async function releaseClaim(claimedFilename) {+ if (!claimedFilename.endsWith(".processing")) return null;+ const dir = getPendingDir();+ const restored = pendingFromProcessingFilename(claimedFilename);+ try {+ await rename(join(dir, claimedFilename), join(dir, restored));+ return restored;+ } catch {+ return null;+ }+}+/*** Remove a pending entry after successful replay.*/@@ -349,9 +367,16 @@ export async function cleanStale() {** @param {Function} fetchJSON - the configured fetchJSON from makeFetchJSON* @param {Function} log - logger function+ * @param {object} [options]+ * @param {boolean} [options.consumeRetries=true] - when false, retryable+ * failures release their claim instead of incrementing the retry count, so+ * a background drainer can keep retrying a transient failure without+ * burning the session-start retry budget. Exhausted and non-retryable+ * entries are deleted exactly as in the default mode.* @returns {{ replayed: number, failed: number, skipped: number, deferred: number }}*/-export async function replayPending(fetchJSON, log) {+export async function replayPending(fetchJSON, log, options = {}) {+ const consumeRetries = options.consumeRetries !== false;const pending = await listPending();if (pending.length === 0) {@@ -426,7 +451,18 @@ export async function replayPending(fetchJSON, log) {await dequeue(claimedFilename);skipped++;} else {- await incrementRetry(claimedFilename, entry);+ if (consumeRetries) {+ await incrementRetry(claimedFilename, entry);+ } else {+ const released = await releaseClaim(claimedFilename);+ log("pending-queue", {+ action: released ? "replay-deferred" : "release-failed",+ sessionId: entry.sessionId,+ type: entry.type,+ status: res?.result?.status || res?.status,+ retries: entry.retries || 0,+ });+ }failed++;if (entry.type === "addMessage") {deferred += Math.max(0, pending.length - processed);diff --git a/examples/opencode-plugin/lib/shared/pending-queue.mjs b/examples/opencode-plugin/lib/shared/pending-queue.mjsindex 4b3836a4b..38dccb1c8 100644--- a/examples/opencode-plugin/lib/shared/pending-queue.mjs+++ b/examples/opencode-plugin/lib/shared/pending-queue.mjs@@ -4,9 +4,10 @@** When the OpenViking server is temporarily unreachable, write operations* (addMessage, commitSession) serialize their payloads to- * `~/.openviking/pending/` as JSON files. On the next session-start, the- * queue is replayed in small batches. This is a session-start-triggered retry- * path with maxRetries/TTL, not a long-running background worker.+ * `~/.openviking/pending/` as JSON files. The queue is replayed in small+ * batches, either at session-start (consuming retry budgets, with+ * maxRetries/TTL) or by a long-running drainer that passes+ * `consumeRetries: false` so transient failures stay retryable.** Each file contains: { type, sessionId, payload, createdAt, retries, dedupKey }*@@ -274,6 +275,23 @@ export async function claimForReplay(filename) {}}+/**+ * Release a claimed file back to the queue without consuming a retry. The+ * entry keeps its original filename, retry count, and createdAt position, so a+ * later run retries it as if this attempt never happened.+ */+export async function releaseClaim(claimedFilename) {+ if (!claimedFilename.endsWith(".processing")) return null;+ const dir = getPendingDir();+ const restored = pendingFromProcessingFilename(claimedFilename);+ try {+ await rename(join(dir, claimedFilename), join(dir, restored));+ return restored;+ } catch {+ return null;+ }+}+/*** Remove a pending entry after successful replay.*/@@ -350,9 +368,16 @@ export async function cleanStale() {** @param {Function} fetchJSON - the configured fetchJSON from makeFetchJSON* @param {Function} log - logger function+ * @param {object} [options]+ * @param {boolean} [options.consumeRetries=true] - when false, retryable+ * failures release their claim instead of incrementing the retry count, so+ * a background drainer can keep retrying a transient failure without+ * burning the session-start retry budget. Exhausted and non-retryable+ * entries are deleted exactly as in the default mode.* @returns {{ replayed: number, failed: number, skipped: number, deferred: number }}*/-export async function replayPending(fetchJSON, log) {+export async function replayPending(fetchJSON, log, options = {}) {+ const consumeRetries = options.consumeRetries !== false;const pending = await listPending();if (pending.length === 0) {@@ -427,7 +452,18 @@ export async function replayPending(fetchJSON, log) {await dequeue(claimedFilename);skipped++;} else {- await incrementRetry(claimedFilename, entry);+ if (consumeRetries) {+ await incrementRetry(claimedFilename, entry);+ } else {+ const released = await releaseClaim(claimedFilename);+ log("pending-queue", {+ action: released ? "replay-deferred" : "release-failed",+ sessionId: entry.sessionId,+ type: entry.type,+ status: res?.result?.status || res?.status,+ retries: entry.retries || 0,+ });+ }failed++;if (entry.type === "addMessage") {deferred += Math.max(0, pending.length - processed);diff --git a/examples/pi-coding-agent-extension/shared/pending-queue.mjs b/examples/pi-coding-agent-extension/shared/pending-queue.mjsindex 4b3836a4b..38dccb1c8 100644--- a/examples/pi-coding-agent-extension/shared/pending-queue.mjs+++ b/examples/pi-coding-agent-extension/shared/pending-queue.mjs@@ -4,9 +4,10 @@** When the OpenViking server is temporarily unreachable, write operations* (addMessage, commitSession) serialize their payloads to- * `~/.openviking/pending/` as JSON files. On the next session-start, the- * queue is replayed in small batches. This is a session-start-triggered retry- * path with maxRetries/TTL, not a long-running background worker.+ * `~/.openviking/pending/` as JSON files. The queue is replayed in small+ * batches, either at session-start (consuming retry budgets, with+ * maxRetries/TTL) or by a long-running drainer that passes+ * `consumeRetries: false` so transient failures stay retryable.** Each file contains: { type, sessionId, payload, createdAt, retries, dedupKey }*@@ -274,6 +275,23 @@ export async function claimForReplay(filename) {}}+/**+ * Release a claimed file back to the queue without consuming a retry. The+ * entry keeps its original filename, retry count, and createdAt position, so a+ * later run retries it as if this attempt never happened.+ */+export async function releaseClaim(claimedFilename) {+ if (!claimedFilename.endsWith(".processing")) return null;+ const dir = getPendingDir();+ const restored = pendingFromProcessingFilename(claimedFilename);+ try {+ await rename(join(dir, claimedFilename), join(dir, restored));+ return restored;+ } catch {+ return null;+ }+}+/*** Remove a pending entry after successful replay.*/@@ -350,9 +368,16 @@ export async function cleanStale() {** @param {Function} fetchJSON - the configured fetchJSON from makeFetchJSON* @param {Function} log - logger function+ * @param {object} [options]+ * @param {boolean} [options.consumeRetries=true] - when false, retryable+ * failures release their claim instead of incrementing the retry count, so+ * a background drainer can keep retrying a transient failure without+ * burning the session-start retry budget. Exhausted and non-retryable+ * entries are deleted exactly as in the default mode.* @returns {{ replayed: number, failed: number, skipped: number, deferred: number }}*/-export async function replayPending(fetchJSON, log) {+export async function replayPending(fetchJSON, log, options = {}) {+ const consumeRetries = options.consumeRetries !== false;const pending = await listPending();if (pending.length === 0) {@@ -427,7 +452,18 @@ export async function replayPending(fetchJSON, log) {await dequeue(claimedFilename);skipped++;} else {- await incrementRetry(claimedFilename, entry);+ if (consumeRetries) {+ await incrementRetry(claimedFilename, entry);+ } else {+ const released = await releaseClaim(claimedFilename);+ log("pending-queue", {+ action: released ? "replay-deferred" : "release-failed",+ sessionId: entry.sessionId,+ type: entry.type,+ status: res?.result?.status || res?.status,+ retries: entry.retries || 0,+ });+ }failed++;if (entry.type === "addMessage") {deferred += Math.max(0, pending.length - processed);diff --git a/examples/zcode-memory-plugin/scripts/shared/pending-queue.mjs b/examples/zcode-memory-plugin/scripts/shared/pending-queue.mjsindex 4b3836a4b..38dccb1c8 100644--- a/examples/zcode-memory-plugin/scripts/shared/pending-queue.mjs+++ b/examples/zcode-memory-plugin/scripts/shared/pending-queue.mjs@@ -4,9 +4,10 @@** When the OpenViking server is temporarily unreachable, write operations* (addMessage, commitSession) serialize their payloads to- * `~/.openviking/pending/` as JSON files. On the next session-start, the- * queue is replayed in small batches. This is a session-start-triggered retry- * path with maxRetries/TTL, not a long-running background worker.+ * `~/.openviking/pending/` as JSON files. The queue is replayed in small+ * batches, either at session-start (consuming retry budgets, with+ * maxRetries/TTL) or by a long-running drainer that passes+ * `consumeRetries: false` so transient failures stay retryable.** Each file contains: { type, sessionId, payload, createdAt, retries, dedupKey }*@@ -274,6 +275,23 @@ export async function claimForReplay(filename) {}}+/**+ * Release a claimed file back to the queue without consuming a retry. The+ * entry keeps its original filename, retry count, and createdAt position, so a+ * later run retries it as if this attempt never happened.+ */+export async function releaseClaim(claimedFilename) {+ if (!claimedFilename.endsWith(".processing")) return null;+ const dir = getPendingDir();+ const restored = pendingFromProcessingFilename(claimedFilename);+ try {+ await rename(join(dir, claimedFilename), join(dir, restored));+ return restored;+ } catch {+ return null;+ }+}+/*** Remove a pending entry after successful replay.*/@@ -350,9 +368,16 @@ export async function cleanStale() {** @param {Function} fetchJSON - the configured fetchJSON from makeFetchJSON* @param {Function} log - logger function+ * @param {object} [options]+ * @param {boolean} [options.consumeRetries=true] - when false, retryable+ * failures release their claim instead of incrementing the retry count, so+ * a background drainer can keep retrying a transient failure without+ * burning the session-start retry budget. Exhausted and non-retryable+ * entries are deleted exactly as in the default mode.* @returns {{ replayed: number, failed: number, skipped: number, deferred: number }}*/-export async function replayPending(fetchJSON, log) {+export async function replayPending(fetchJSON, log, options = {}) {+ const consumeRetries = options.consumeRetries !== false;const pending = await listPending();if (pending.length === 0) {@@ -427,7 +452,18 @@ export async function replayPending(fetchJSON, log) {await dequeue(claimedFilename);skipped++;} else {- await incrementRetry(claimedFilename, entry);+ if (consumeRetries) {+ await incrementRetry(claimedFilename, entry);+ } else {+ const released = await releaseClaim(claimedFilename);+ log("pending-queue", {+ action: released ? "replay-deferred" : "release-failed",+ sessionId: entry.sessionId,+ type: entry.type,+ status: res?.result?.status || res?.status,+ retries: entry.retries || 0,+ });+ }failed++;if (entry.type === "addMessage") {deferred += Math.max(0, pending.length - processed);