Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(plugins): drain the dsh pending queue in-process so a transient write failure self-heals (#4779)

Nick James3841e6f2992b
diff --git a/examples/claude-code-memory-plugin/scripts/shared/pending-queue.mjs b/examples/claude-code-memory-plugin/scripts/shared/pending-queue.mjs
index 4b3836a4b..38dccb1c8 100644
--- a/examples/claude-code-memory-plugin/scripts/shared/pending-queue.mjs
+++ b/examples/claude-code-memory-plugin/scripts/shared/pending-queue.mjs
@@ -4,9 +4,10 @@
*
* When the OpenViking server is temporarily unreachable, write operations
* (addMessage, commitSession) serialize their payloads to
- * `~/.openviking/pending/` as JSON files. On the next session-start, the
- * queue is replayed in small batches. This is a session-start-triggered retry
- * path with maxRetries/TTL, not a long-running background worker.
+ * `~/.openviking/pending/` as JSON files. The queue is replayed in small
+ * batches, either at session-start (consuming retry budgets, with
+ * maxRetries/TTL) or by a long-running drainer that passes
+ * `consumeRetries: false` so transient failures stay retryable.
*
* Each file contains: { type, sessionId, payload, createdAt, retries, dedupKey }
*
@@ -274,6 +275,23 @@ export async function claimForReplay(filename) {
}
}
+/**
+ * Release a claimed file back to the queue without consuming a retry. The
+ * entry keeps its original filename, retry count, and createdAt position, so a
+ * later run retries it as if this attempt never happened.
+ */
+export async function releaseClaim(claimedFilename) {
+ if (!claimedFilename.endsWith(".processing")) return null;
+ const dir = getPendingDir();
+ const restored = pendingFromProcessingFilename(claimedFilename);
+ try {
+ await rename(join(dir, claimedFilename), join(dir, restored));
+ return restored;
+ } catch {
+ return null;
+ }
+}
+
/**
* Remove a pending entry after successful replay.
*/
@@ -350,9 +368,16 @@ export async function cleanStale() {
*
* @param {Function} fetchJSON - the configured fetchJSON from makeFetchJSON
* @param {Function} log - logger function
+ * @param {object} [options]
+ * @param {boolean} [options.consumeRetries=true] - when false, retryable
+ * failures release their claim instead of incrementing the retry count, so
+ * a background drainer can keep retrying a transient failure without
+ * burning the session-start retry budget. Exhausted and non-retryable
+ * entries are deleted exactly as in the default mode.
* @returns {{ replayed: number, failed: number, skipped: number, deferred: number }}
*/
-export async function replayPending(fetchJSON, log) {
+export async function replayPending(fetchJSON, log, options = {}) {
+ const consumeRetries = options.consumeRetries !== false;
const pending = await listPending();
if (pending.length === 0) {
@@ -427,7 +452,18 @@ export async function replayPending(fetchJSON, log) {
await dequeue(claimedFilename);
skipped++;
} else {
- await incrementRetry(claimedFilename, entry);
+ if (consumeRetries) {
+ await incrementRetry(claimedFilename, entry);
+ } else {
+ const released = await releaseClaim(claimedFilename);
+ log("pending-queue", {
+ action: released ? "replay-deferred" : "release-failed",
+ sessionId: entry.sessionId,
+ type: entry.type,
+ status: res?.result?.status || res?.status,
+ retries: entry.retries || 0,
+ });
+ }
failed++;
if (entry.type === "addMessage") {
deferred += Math.max(0, pending.length - processed);
diff --git a/examples/codex-memory-plugin/scripts/shared/pending-queue.mjs b/examples/codex-memory-plugin/scripts/shared/pending-queue.mjs
index 4b3836a4b..38dccb1c8 100644
--- a/examples/codex-memory-plugin/scripts/shared/pending-queue.mjs
+++ b/examples/codex-memory-plugin/scripts/shared/pending-queue.mjs
@@ -4,9 +4,10 @@
*
* When the OpenViking server is temporarily unreachable, write operations
* (addMessage, commitSession) serialize their payloads to
- * `~/.openviking/pending/` as JSON files. On the next session-start, the
- * queue is replayed in small batches. This is a session-start-triggered retry
- * path with maxRetries/TTL, not a long-running background worker.
+ * `~/.openviking/pending/` as JSON files. The queue is replayed in small
+ * batches, either at session-start (consuming retry budgets, with
+ * maxRetries/TTL) or by a long-running drainer that passes
+ * `consumeRetries: false` so transient failures stay retryable.
*
* Each file contains: { type, sessionId, payload, createdAt, retries, dedupKey }
*
@@ -274,6 +275,23 @@ export async function claimForReplay(filename) {
}
}
+/**
+ * Release a claimed file back to the queue without consuming a retry. The
+ * entry keeps its original filename, retry count, and createdAt position, so a
+ * later run retries it as if this attempt never happened.
+ */
+export async function releaseClaim(claimedFilename) {
+ if (!claimedFilename.endsWith(".processing")) return null;
+ const dir = getPendingDir();
+ const restored = pendingFromProcessingFilename(claimedFilename);
+ try {
+ await rename(join(dir, claimedFilename), join(dir, restored));
+ return restored;
+ } catch {
+ return null;
+ }
+}
+
/**
* Remove a pending entry after successful replay.
*/
@@ -350,9 +368,16 @@ export async function cleanStale() {
*
* @param {Function} fetchJSON - the configured fetchJSON from makeFetchJSON
* @param {Function} log - logger function
+ * @param {object} [options]
+ * @param {boolean} [options.consumeRetries=true] - when false, retryable
+ * failures release their claim instead of incrementing the retry count, so
+ * a background drainer can keep retrying a transient failure without
+ * burning the session-start retry budget. Exhausted and non-retryable
+ * entries are deleted exactly as in the default mode.
* @returns {{ replayed: number, failed: number, skipped: number, deferred: number }}
*/
-export async function replayPending(fetchJSON, log) {
+export async function replayPending(fetchJSON, log, options = {}) {
+ const consumeRetries = options.consumeRetries !== false;
const pending = await listPending();
if (pending.length === 0) {
@@ -427,7 +452,18 @@ export async function replayPending(fetchJSON, log) {
await dequeue(claimedFilename);
skipped++;
} else {
- await incrementRetry(claimedFilename, entry);
+ if (consumeRetries) {
+ await incrementRetry(claimedFilename, entry);
+ } else {
+ const released = await releaseClaim(claimedFilename);
+ log("pending-queue", {
+ action: released ? "replay-deferred" : "release-failed",
+ sessionId: entry.sessionId,
+ type: entry.type,
+ status: res?.result?.status || res?.status,
+ retries: entry.retries || 0,
+ });
+ }
failed++;
if (entry.type === "addMessage") {
deferred += Math.max(0, pending.length - processed);
diff --git a/examples/dsh-memory-plugin/README.md b/examples/dsh-memory-plugin/README.md
index 08b560766..0d9a8cb09 100644
--- a/examples/dsh-memory-plugin/README.md
+++ b/examples/dsh-memory-plugin/README.md
@@ -150,8 +150,8 @@ The patch can also carry plugin config:
- `session/event` captures user, assistant, and optionally tool-result messages without scraping a transcript.
- `turn/end` checks the OpenViking pending-token threshold and commits when required.
- `skipSubagentSessions: true` excludes sessions marked with `header.origin: subagent` from automatic profile, recall, capture, and commit; it defaults to `false`.
-- `syncTurns: false` stops every write: no captured messages, no threshold or shutdown commit, and no replay of writes an earlier session queued — those stay queued for a session that still writes. Profile injection and recall are unaffected; it defaults to `true`.
-- Failed writes enter the shared OpenViking pending queue for replay at the next session start.
+- `syncTurns: false` stops every new write: no captured messages, no threshold or shutdown commit. Writes queued while the toggle was on are still replayed by the background drainer once the server recovers — they were captured with the toggle on. Profile injection and recall are unaffected; it defaults to `true`.
+- Failed writes enter the shared OpenViking pending queue. A background drainer (default every 60s, `OPENVIKING_PENDING_DRAIN_INTERVAL_MS`) probes the server health and replays the queue in-process, so a transient write failure recovers without restarting dsh; it does not consume the session-start retry budget. Session-start replays keep consuming retries as before.
- `tools/pre-execute` blocks DSH filesystem and shell tools from treating `viking://` URIs as local paths, pointing the model at the bridged `mcp__openviking__*` tools instead.
Each DSH session maps to `dsh-<session-id>` in OpenViking. Workspace-derived actor peers are resolved per session and sent on every session-specific request: the peer is the git identity of the session's workspace — the normalized `origin` URL (`[email protected]:volcengine/OpenViking.git` becomes `github.com-volcengine-openviking`), else the repository root path, that fallback keeping the older rule where every non-letter-or-digit character becomes `-`. Outside a git repository no peer is sent at all, and what is remembered there goes to the user-level space `viking://user/<you>/memories`. One repository therefore keeps one peer across subdirectories, worktrees, clones and machines, while a fork's different origin keeps it separate. DSH does not read workspace `.openviking/config.json` files, so a `peer.id` written there has no effect; pin a peer with `OPENVIKING_PEER_ID` instead. Memories written under the older path-derived peer stay reachable: the default `recallPeerScope: all` sweeps every peer under the user.
diff --git a/examples/dsh-memory-plugin/index.mjs b/examples/dsh-memory-plugin/index.mjs
index 69d6623cd..f6a738efd 100644
--- a/examples/dsh-memory-plugin/index.mjs
+++ b/examples/dsh-memory-plugin/index.mjs
@@ -21,6 +21,14 @@ export function apply(ctx, input = {}) {
() => () => runtime.disposeAll(),
"openvikingMemory.disposeAll()",
);
+ // The pending-queue drainer is the in-process recovery path: without it a
+ // single transient write failure latches capture/commit until the next dsh
+ // restart. Started here so every session shares one single-flight drainer.
+ runtime.startDrainer();
+ ctx.effect(
+ () => () => runtime.stopDrainer(),
+ "openvikingMemory.stopDrainer()",
+ );
ctx.on("agent/session-start", ({ agent }) => {
if (skipMemory(agent.session)) return false;
diff --git a/examples/dsh-memory-plugin/pending-queue.test.mjs b/examples/dsh-memory-plugin/pending-queue.test.mjs
new file mode 100644
index 000000000..1173a6a18
--- /dev/null
+++ b/examples/dsh-memory-plugin/pending-queue.test.mjs
@@ -0,0 +1,180 @@
+import assert from "node:assert/strict";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import test from "node:test";
+
+import {
+ enqueue,
+ listPending,
+ replayPending,
+} from "./shared/pending-queue.mjs";
+
+const originalEnv = {
+ dir: process.env.OPENVIKING_PENDING_DIR,
+ maxRetries: process.env.OPENVIKING_PENDING_MAX_RETRIES,
+ replayLimit: process.env.OPENVIKING_PENDING_REPLAY_LIMIT,
+ ttlDays: process.env.OPENVIKING_PENDING_TTL_DAYS,
+};
+
+async function withPendingDir(fn) {
+ const dir = await mkdtemp(join(tmpdir(), "dsh-pending-test-"));
+ process.env.OPENVIKING_PENDING_DIR = dir;
+ delete process.env.OPENVIKING_PENDING_MAX_RETRIES;
+ delete process.env.OPENVIKING_PENDING_REPLAY_LIMIT;
+ delete process.env.OPENVIKING_PENDING_TTL_DAYS;
+ try {
+ return await fn(dir);
+ } finally {
+ if (originalEnv.dir === undefined) delete process.env.OPENVIKING_PENDING_DIR;
+ else process.env.OPENVIKING_PENDING_DIR = originalEnv.dir;
+ if (originalEnv.maxRetries === undefined) delete process.env.OPENVIKING_PENDING_MAX_RETRIES;
+ else process.env.OPENVIKING_PENDING_MAX_RETRIES = originalEnv.maxRetries;
+ if (originalEnv.replayLimit === undefined) delete process.env.OPENVIKING_PENDING_REPLAY_LIMIT;
+ else process.env.OPENVIKING_PENDING_REPLAY_LIMIT = originalEnv.replayLimit;
+ if (originalEnv.ttlDays === undefined) delete process.env.OPENVIKING_PENDING_TTL_DAYS;
+ else process.env.OPENVIKING_PENDING_TTL_DAYS = originalEnv.ttlDays;
+ await rm(dir, { recursive: true, force: true });
+ }
+}
+
+test("replay with consumeRetries:false keeps a retryable failure intact for the next attempt", async () => {
+ await withPendingDir(async () => {
+ const payload = { role: "user", content: "remember this" };
+ await enqueue("addMessage", "dsh-drain", payload);
+
+ let calls = 0;
+ const first = await replayPending(
+ async () => {
+ calls += 1;
+ return { ok: false, status: 503, error: { message: "unavailable" } };
+ },
+ () => {},
+ { consumeRetries: false },
+ );
+
+ assert.equal(first.replayed, 0);
+ assert.equal(first.failed, 1);
+ assert.equal(calls, 1);
+
+ const pending = await listPending();
+ assert.equal(pending.length, 1);
+ assert.equal(pending[0].entry.retries, 0);
+ assert.ok(pending[0].filename.endsWith("_0.json"), pending[0].filename);
+ assert.deepEqual(pending[0].entry.payload, payload);
+
+ const recovered = await replayPending(
+ async () => ({ ok: true }),
+ () => {},
+ { consumeRetries: false },
+ );
+ assert.equal(recovered.replayed, 1);
+ assert.deepEqual(await listPending(), []);
+ });
+});
+
+test("replay with consumeRetries:false still deletes exhausted entries", async () => {
+ await withPendingDir(async () => {
+ process.env.OPENVIKING_PENDING_MAX_RETRIES = "0";
+ await enqueue("addMessage", "dsh-exhausted", { content: "doomed" });
+
+ const result = await replayPending(
+ async () => ({ ok: true }),
+ () => {},
+ { consumeRetries: false },
+ );
+
+ assert.equal(result.replayed, 0);
+ assert.equal(result.skipped, 1);
+ assert.deepEqual(await listPending(), []);
+ });
+});
+
+test("replay with consumeRetries:false still deletes non-retryable failures", async () => {
+ await withPendingDir(async () => {
+ await enqueue("addMessage", "dsh-rejected", { content: "too large" });
+
+ const result = await replayPending(
+ async () => ({ ok: false, status: 413, error: { message: "too large" } }),
+ () => {},
+ { consumeRetries: false },
+ );
+
+ assert.equal(result.replayed, 0);
+ assert.equal(result.skipped, 1);
+ assert.deepEqual(await listPending(), []);
+ });
+});
+
+test("replay with consumeRetries:false stops at the first retryable failure and preserves order", async () => {
+ await withPendingDir(async () => {
+ await enqueue("addMessage", "dsh-order", { content: "first" });
+ await enqueue("addMessage", "dsh-order", { content: "second" });
+
+ let calls = 0;
+ const result = await replayPending(
+ async () => {
+ calls += 1;
+ return { ok: false, status: 503, error: { message: "unavailable" } };
+ },
+ () => {},
+ { consumeRetries: false },
+ );
+
+ assert.equal(calls, 1);
+ assert.equal(result.failed, 1);
+ const pending = await listPending();
+ assert.deepEqual(
+ pending.map(item => item.entry.payload.content),
+ ["first", "second"],
+ );
+ assert.equal(pending[0].entry.retries, 0);
+ });
+});
+
+test("replay with consumeRetries:false keeps going after a failed commitSession", async () => {
+ await withPendingDir(async () => {
+ await enqueue("commitSession", "dsh-commit", { keep_recent_count: 10 });
+ await enqueue("addMessage", "dsh-commit", { content: "after the commit" });
+
+ let calls = 0;
+ const result = await replayPending(
+ async (_path, init) => {
+ calls += 1;
+ if (init?.body && init.body.includes("keep_recent_count")) {
+ return { ok: false, status: 503, error: { message: "unavailable" } };
+ }
+ return { ok: true };
+ },
+ () => {},
+ { consumeRetries: false },
+ );
+
+ assert.equal(calls, 2);
+ assert.equal(result.failed, 1);
+ assert.equal(result.replayed, 1);
+ const pending = await listPending();
+ assert.deepEqual(
+ pending.map(item => item.entry.type),
+ ["commitSession"],
+ );
+ assert.equal(pending[0].entry.retries, 0);
+ });
+});
+
+test("replay still consumes retries by default", async () => {
+ await withPendingDir(async () => {
+ await enqueue("addMessage", "dsh-default", { content: "flaky" });
+
+ const result = await replayPending(
+ async () => ({ ok: false, status: 503, error: { message: "unavailable" } }),
+ () => {},
+ );
+
+ assert.equal(result.failed, 1);
+ const pending = await listPending();
+ assert.equal(pending.length, 1);
+ assert.equal(pending[0].entry.retries, 1);
+ assert.ok(pending[0].filename.endsWith("_1.json"), pending[0].filename);
+ });
+});
diff --git a/examples/dsh-memory-plugin/runtime-drain.test.mjs b/examples/dsh-memory-plugin/runtime-drain.test.mjs
new file mode 100644
index 000000000..f1d9a0e55
--- /dev/null
+++ b/examples/dsh-memory-plugin/runtime-drain.test.mjs
@@ -0,0 +1,335 @@
+import assert from "node:assert/strict";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, test } from "node:test";
+import { enqueue, listPending } from "./shared/pending-queue.mjs";
+import { OpenVikingRuntime } from "./runtime.mjs";
+
+const originalPendingDir = process.env.OPENVIKING_PENDING_DIR;
+const originalDrainInterval = process.env.OPENVIKING_PENDING_DRAIN_INTERVAL_MS;
+const tempDirs = [];
+
+afterEach(async () => {
+ if (originalPendingDir === undefined) delete process.env.OPENVIKING_PENDING_DIR;
+ else process.env.OPENVIKING_PENDING_DIR = originalPendingDir;
+ if (originalDrainInterval === undefined) delete process.env.OPENVIKING_PENDING_DRAIN_INTERVAL_MS;
+ else process.env.OPENVIKING_PENDING_DRAIN_INTERVAL_MS = originalDrainInterval;
+ await Promise.all(tempDirs.splice(0).map(dir => rm(dir, { recursive: true, force: true })));
+});
+
+test("drainTick sends queued messages once the server recovers and clears the latch", async () => {
+ const pendingDir = await mkdtemp(join(tmpdir(), "dsh-drain-"));
+ tempDirs.push(pendingDir);
+ process.env.OPENVIKING_PENDING_DIR = pendingDir;
+
+ let healthy = false;
+ let addCalls = 0;
+ const runtime = new OpenVikingRuntime({
+ async addMessage() {
+ addCalls += 1;
+ return healthy
+ ? { ok: true }
+ : { ok: false, status: 503, error: { message: "down" } };
+ },
+ async fetchJSON() {
+ return healthy
+ ? { ok: true }
+ : { ok: false, status: 503, error: { message: "down" } };
+ },
+ async healthResult() {
+ return healthy
+ ? { ok: true }
+ : { ok: false, status: 503, error: { message: "down" } };
+ },
+ }, config(), { debug() {} });
+ const session = { id: "recover", header: { cwd: "/workspace" } };
+ runtime.stateFor(session).ready = true;
+
+ runtime.capture(session, userEvent("Queued during the outage."));
+ await runtime.flush(session);
+ assert.equal((await listPending()).length, 1);
+ assert.equal(runtime.stateFor(session).hasPendingWrites, true);
+
+ healthy = true;
+ await runtime.drainTick();
+
+ assert.equal((await listPending()).length, 0);
+ assert.equal(runtime.stateFor(session).hasPendingWrites, false);
+
+ runtime.capture(session, userEvent("Sent directly after recovery."));
+ await runtime.flush(session);
+ assert.equal(addCalls, 2);
+ assert.equal((await listPending()).length, 0);
+});
+
+test("drainTick keeps the latch while the server stays down and leaves the entry retryable", async () => {
+ const pendingDir = await mkdtemp(join(tmpdir(), "dsh-drain-down-"));
+ tempDirs.push(pendingDir);
+ process.env.OPENVIKING_PENDING_DIR = pendingDir;
+
+ let addCalls = 0;
+ const runtime = new OpenVikingRuntime({
+ async addMessage() {
+ addCalls += 1;
+ return { ok: false, status: 503, error: { message: "down" } };
+ },
+ async fetchJSON() {
+ return { ok: false, status: 503, error: { message: "down" } };
+ },
+ async healthResult() {
+ return { ok: false, status: 503, error: { message: "down" } };
+ },
+ }, config(), { debug() {} });
+ const session = { id: "still-down", header: { cwd: "/workspace" } };
+ runtime.stateFor(session).ready = true;
+
+ runtime.capture(session, userEvent("First queued message."));
+ await runtime.flush(session);
+ await runtime.drainTick();
+
+ assert.equal(runtime.stateFor(session).hasPendingWrites, true);
+ const pending = await listPending();
+ assert.equal(pending.length, 1);
+ assert.equal(pending[0].entry.retries, 0);
+
+ runtime.capture(session, userEvent("Second queued message."));
+ await runtime.flush(session);
+ assert.equal(addCalls, 1);
+ assert.equal((await listPending()).length, 2);
+});
+
+test("drainTick does not hit the network while the queue is empty", async () => {
+ let fetchCalls = 0;
+ const runtime = new OpenVikingRuntime({
+ async fetchJSON() {
+ fetchCalls += 1;
+ return { ok: true };
+ },
+ }, config(), { debug() {} });
+ runtime.stateFor({ id: "idle", header: { cwd: "/workspace" } });
+
+ await runtime.drainTick();
+
+ assert.equal(fetchCalls, 0);
+});
+
+test("commit resumes once the drain clears the backlog", async () => {
+ const pendingDir = await mkdtemp(join(tmpdir(), "dsh-commit-resume-"));
+ tempDirs.push(pendingDir);
+ process.env.OPENVIKING_PENDING_DIR = pendingDir;
+
+ let healthy = false;
+ let commitCalls = 0;
+ const runtime = new OpenVikingRuntime({
+ async addMessage() {
+ return healthy
+ ? { ok: true }
+ : { ok: false, status: 503, error: { message: "down" } };
+ },
+ async fetchJSON() {
+ return healthy
+ ? { ok: true }
+ : { ok: false, status: 503, error: { message: "down" } };
+ },
+ async healthResult() {
+ return healthy
+ ? { ok: true }
+ : { ok: false, status: 503, error: { message: "down" } };
+ },
+ async getSession() {
+ return { pending_tokens: 20000 };
+ },
+ async commitSession() {
+ commitCalls += 1;
+ return { ok: true };
+ },
+ }, config(), { debug() {} });
+ const session = { id: "commit-resume", header: { cwd: "/workspace" } };
+ const state = runtime.stateFor(session);
+ state.ready = true;
+
+ runtime.capture(session, userEvent("Queued during the outage."));
+ await runtime.flush(session);
+ assert.equal(state.hasPendingWrites, true);
+
+ // While the backlog holds, the commit check keeps its ordering guard.
+ runtime.maybeCommit(session, { type: "turn/end" });
+ await runtime.flush(session);
+ assert.equal(commitCalls, 0);
+
+ // The drainer clears the backlog, and the next turn/end commits again.
+ healthy = true;
+ await runtime.drainTick();
+ assert.equal(state.hasPendingWrites, false);
+
+ runtime.maybeCommit(session, { type: "turn/end" });
+ await runtime.flush(session);
+ assert.equal(commitCalls, 1);
+});
+
+test("startDrainer drains the queue on its interval", async t => {
+ t.mock.timers.enable({ apis: ["setInterval"] });
+ const pendingDir = await mkdtemp(join(tmpdir(), "dsh-drain-interval-"));
+ tempDirs.push(pendingDir);
+ process.env.OPENVIKING_PENDING_DIR = pendingDir;
+ process.env.OPENVIKING_PENDING_DRAIN_INTERVAL_MS = "60000";
+
+ let healthy = false;
+ const runtime = new OpenVikingRuntime({
+ async addMessage() {
+ return healthy
+ ? { ok: true }
+ : { ok: false, status: 503, error: { message: "down" } };
+ },
+ async fetchJSON() {
+ return healthy
+ ? { ok: true }
+ : { ok: false, status: 503, error: { message: "down" } };
+ },
+ async healthResult() {
+ return healthy
+ ? { ok: true }
+ : { ok: false, status: 503, error: { message: "down" } };
+ },
+ }, config(), { debug() {} });
+ const session = { id: "interval", header: { cwd: "/workspace" } };
+ runtime.stateFor(session).ready = true;
+
+ runtime.capture(session, userEvent("Queued."));
+ await runtime.flush(session);
+ assert.equal((await listPending()).length, 1);
+
+ runtime.startDrainer();
+ healthy = true;
+ t.mock.timers.tick(60000);
+ await runtime.drainPromise;
+
+ assert.equal((await listPending()).length, 0);
+ assert.equal(runtime.stateFor(session).hasPendingWrites, false);
+
+ runtime.stopDrainer();
+});
+
+test("drainTick runs single-flight", async () => {
+ const pendingDir = await mkdtemp(join(tmpdir(), "dsh-drain-single-"));
+ tempDirs.push(pendingDir);
+ process.env.OPENVIKING_PENDING_DIR = pendingDir;
+ await enqueue("addMessage", "dsh-single", { content: "held" });
+
+ let release;
+ let startedResolve;
+ const started = new Promise(resolve => {
+ startedResolve = resolve;
+ });
+ let fetchCalls = 0;
+ const runtime = new OpenVikingRuntime({
+ async healthResult() {
+ return { ok: true };
+ },
+ async fetchJSON() {
+ fetchCalls += 1;
+ startedResolve();
+ await new Promise(resolve => {
+ release = resolve;
+ });
+ return { ok: true };
+ },
+ }, config(), { debug() {} });
+ runtime.stateFor({ id: "single", header: { cwd: "/workspace" } }).ready = true;
+
+ const first = runtime.drainTick();
+ await started;
+ await runtime.drainTick();
+ release();
+ await first;
+
+ assert.equal(fetchCalls, 1);
+});
+
+test("drainTick derives each session's latch from its own entries", async () => {
+ const pendingDir = await mkdtemp(join(tmpdir(), "dsh-drain-isolate-"));
+ tempDirs.push(pendingDir);
+ process.env.OPENVIKING_PENDING_DIR = pendingDir;
+
+ const runtime = new OpenVikingRuntime({
+ async addMessage() {
+ return { ok: false, status: 503, error: { message: "down" } };
+ },
+ async healthResult() {
+ return { ok: true };
+ },
+ async fetchJSON() {
+ return { ok: false, status: 503, error: { message: "down" } };
+ },
+ }, config(), { debug() {} });
+ const backlogged = { id: "backlogged", header: { cwd: "/workspace" } };
+ const clean = { id: "clean", header: { cwd: "/workspace" } };
+ runtime.stateFor(backlogged).ready = true;
+ runtime.stateFor(clean).ready = true;
+
+ runtime.capture(backlogged, userEvent("Queued."));
+ await runtime.flush(backlogged);
+ await runtime.drainTick();
+
+ assert.equal(runtime.stateFor(backlogged).hasPendingWrites, true);
+ assert.equal(runtime.stateFor(clean).hasPendingWrites, false);
+});
+
+test("startDrainer is idempotent, stopDrainer is safe, and an invalid env falls back to 60s", async t => {
+ t.mock.timers.enable({ apis: ["setInterval"] });
+ const pendingDir = await mkdtemp(join(tmpdir(), "dsh-drain-fallback-"));
+ tempDirs.push(pendingDir);
+ process.env.OPENVIKING_PENDING_DIR = pendingDir;
+ process.env.OPENVIKING_PENDING_DRAIN_INTERVAL_MS = "not-a-number";
+ await enqueue("addMessage", "dsh-fallback", { content: "held" });
+
+ let fetchCalls = 0;
+ const runtime = new OpenVikingRuntime({
+ async healthResult() {
+ return { ok: true };
+ },
+ async fetchJSON() {
+ fetchCalls += 1;
+ return { ok: true };
+ },
+ }, config(), { debug() {} });
+ runtime.stateFor({ id: "fallback", header: { cwd: "/workspace" } }).ready = true;
+
+ const timer = runtime.startDrainer();
+ assert.equal(runtime.startDrainer(), timer);
+ t.mock.timers.tick(59999);
+ assert.equal(fetchCalls, 0);
+ t.mock.timers.tick(1);
+ await runtime.drainPromise;
+ assert.ok(fetchCalls >= 1);
+
+ runtime.stopDrainer();
+ runtime.stopDrainer();
+});
+
+function config() {
+ return {
+ explicitPeerId: "",
+ workspacePeer: false,
+ peerId: "",
+ syncTurns: true,
+ captureAssistantTurns: true,
+ captureToolResults: false,
+ captureToolMaxChars: 1000000,
+ captureMaxLength: 24000,
+ captureMode: "semantic",
+ commitKeepRecentCount: 10,
+ };
+}
+
+function userEvent(text) {
+ return {
+ type: "user/message",
+ data: {
+ role: "user",
+ content: [{ type: "text", text }],
+ source: { kind: "user" },
+ },
+ };
+}
diff --git a/examples/dsh-memory-plugin/runtime.mjs b/examples/dsh-memory-plugin/runtime.mjs
index 1dc9f8137..0449b2b67 100644
--- a/examples/dsh-memory-plugin/runtime.mjs
+++ b/examples/dsh-memory-plugin/runtime.mjs
@@ -22,6 +22,10 @@ export class OpenVikingRuntime {
this.config = config;
this.logger = logger;
this.states = new Map();
+ this.drainTimer = null;
+ this.drainRunning = false;
+ this.drainPromise = Promise.resolve();
+ this.drainHealth = null;
}
stateFor(session) {
@@ -90,10 +94,7 @@ export class OpenVikingRuntime {
// Replay is a write, so it stays behind the same toggle: a backlog queued
// while capture was on waits for a session that still writes.
if (state.config.syncTurns) {
- await replayPending(
- (path, init) => this.client.fetchJSON(path, init),
- (stage, data) => this.log(stage, data),
- );
+ await this.replayPendingQueue();
}
await this.refreshPendingState(state);
const profile = await buildProfileBlock(
@@ -260,7 +261,15 @@ export class OpenVikingRuntime {
const createdAt = Math.max(Date.now(), state.pendingCreatedAt + 1);
state.pendingCreatedAt = createdAt;
const result = await enqueue(type, state.ovSessionId, payload, { createdAt });
- if (result.ok) state.hasPendingWrites = true;
+ if (result.ok) {
+ // Log the latch transition only: every message that follows while the
+ // latch holds takes the cheap enqueue path, so this fires once per
+ // outage, not once per message.
+ if (!state.hasPendingWrites) {
+ this.log("pending_latched", { sessionId: state.ovSessionId, type });
+ }
+ state.hasPendingWrites = true;
+ }
if (!result.ok) {
this.log("pending_enqueue_error", {
sessionId: state.ovSessionId,
@@ -295,6 +304,7 @@ export class OpenVikingRuntime {
}
async refreshPendingState(state) {
+ const wasPending = state.hasPendingWrites;
const pending = (await listPending()).filter(
item => item.entry?.sessionId === state.ovSessionId,
);
@@ -303,6 +313,84 @@ export class OpenVikingRuntime {
(latest, item) => Math.max(latest, Number(item.entry?.createdAt || 0)),
state.pendingCreatedAt,
);
+ // Only the flip back to direct sends is logged: the recovery moment that
+ // proves the drainer worked, once per outage.
+ if (wasPending && !state.hasPendingWrites) {
+ this.log("pending_cleared", { sessionId: state.ovSessionId });
+ }
+ }
+
+ /**
+ * Replay the pending queue through this runtime's client. The session-start
+ * path calls it without options and keeps consuming retries; the drainer
+ * passes consumeRetries:false so transient failures stay retryable.
+ */
+ async replayPendingQueue(options = {}) {
+ await replayPending(
+ (path, init) => this.client.fetchJSON(path, init),
+ (stage, data) => this.log(stage, data),
+ options,
+ );
+ }
+
+ /**
+ * One drainer tick, following the session-start replay flow: probe health
+ * first and only replay when the server answers. Replays run without
+ * consuming retry budgets, then every session's latch is re-derived from
+ * the queue. An empty queue clears the latches with zero HTTP traffic, so
+ * once a transient write failure recovers, capture and commit resume on
+ * their own without restarting the long-lived dsh process.
+ */
+ async drainTick() {
+ if (this.drainRunning) return;
+ this.drainRunning = true;
+ try {
+ const pending = await listPending();
+ if (pending.length > 0) {
+ const health = await this.client.healthResult();
+ if (!health.ok) {
+ // Only the flip into the outage is logged, not every 60s probe.
+ if (this.drainHealth !== false) {
+ this.log("drain_health_down", { status: health.status || 0 });
+ }
+ this.drainHealth = false;
+ } else {
+ if (this.drainHealth === false) {
+ this.log("drain_health_restored", {});
+ }
+ this.drainHealth = true;
+ await this.replayPendingQueue({ consumeRetries: false });
+ }
+ }
+ for (const state of this.states.values()) {
+ await this.refreshPendingState(state);
+ }
+ } finally {
+ this.drainRunning = false;
+ }
+ }
+
+ /**
+ * Start the background drainer. The interval is fixed per process; each tick
+ * is single-flight, so a slow replay run never overlaps the next one.
+ */
+ startDrainer() {
+ if (this.drainTimer) return this.drainTimer;
+ const parsed = parseInt(process.env.OPENVIKING_PENDING_DRAIN_INTERVAL_MS || "", 10);
+ const intervalMs = Number.isFinite(parsed) && parsed > 0 ? parsed : 60000;
+ this.drainTimer = setInterval(() => {
+ this.drainPromise = this.drainTick().catch(error => this.log("drain_error", {
+ error: error instanceof Error ? error.message : String(error),
+ }));
+ }, intervalMs);
+ this.drainTimer.unref?.();
+ return this.drainTimer;
+ }
+
+ stopDrainer() {
+ if (!this.drainTimer) return;
+ clearInterval(this.drainTimer);
+ this.drainTimer = null;
}
async flush(session) {
diff --git a/examples/dsh-memory-plugin/shared/pending-queue.mjs b/examples/dsh-memory-plugin/shared/pending-queue.mjs
index 4b3836a4b..38dccb1c8 100644
--- a/examples/dsh-memory-plugin/shared/pending-queue.mjs
+++ b/examples/dsh-memory-plugin/shared/pending-queue.mjs
@@ -4,9 +4,10 @@
*
* When the OpenViking server is temporarily unreachable, write operations
* (addMessage, commitSession) serialize their payloads to
- * `~/.openviking/pending/` as JSON files. On the next session-start, the
- * queue is replayed in small batches. This is a session-start-triggered retry
- * path with maxRetries/TTL, not a long-running background worker.
+ * `~/.openviking/pending/` as JSON files. The queue is replayed in small
+ * batches, either at session-start (consuming retry budgets, with
+ * maxRetries/TTL) or by a long-running drainer that passes
+ * `consumeRetries: false` so transient failures stay retryable.
*
* Each file contains: { type, sessionId, payload, createdAt, retries, dedupKey }
*
@@ -274,6 +275,23 @@ export async function claimForReplay(filename) {
}
}
+/**
+ * Release a claimed file back to the queue without consuming a retry. The
+ * entry keeps its original filename, retry count, and createdAt position, so a
+ * later run retries it as if this attempt never happened.
+ */
+export async function releaseClaim(claimedFilename) {
+ if (!claimedFilename.endsWith(".processing")) return null;
+ const dir = getPendingDir();
+ const restored = pendingFromProcessingFilename(claimedFilename);
+ try {
+ await rename(join(dir, claimedFilename), join(dir, restored));
+ return restored;
+ } catch {
+ return null;
+ }
+}
+
/**
* Remove a pending entry after successful replay.
*/
@@ -350,9 +368,16 @@ export async function cleanStale() {
*
* @param {Function} fetchJSON - the configured fetchJSON from makeFetchJSON
* @param {Function} log - logger function
+ * @param {object} [options]
+ * @param {boolean} [options.consumeRetries=true] - when false, retryable
+ * failures release their claim instead of incrementing the retry count, so
+ * a background drainer can keep retrying a transient failure without
+ * burning the session-start retry budget. Exhausted and non-retryable
+ * entries are deleted exactly as in the default mode.
* @returns {{ replayed: number, failed: number, skipped: number, deferred: number }}
*/
-export async function replayPending(fetchJSON, log) {
+export async function replayPending(fetchJSON, log, options = {}) {
+ const consumeRetries = options.consumeRetries !== false;
const pending = await listPending();
if (pending.length === 0) {
@@ -427,7 +452,18 @@ export async function replayPending(fetchJSON, log) {
await dequeue(claimedFilename);
skipped++;
} else {
- await incrementRetry(claimedFilename, entry);
+ if (consumeRetries) {
+ await incrementRetry(claimedFilename, entry);
+ } else {
+ const released = await releaseClaim(claimedFilename);
+ log("pending-queue", {
+ action: released ? "replay-deferred" : "release-failed",
+ sessionId: entry.sessionId,
+ type: entry.type,
+ status: res?.result?.status || res?.status,
+ retries: entry.retries || 0,
+ });
+ }
failed++;
if (entry.type === "addMessage") {
deferred += Math.max(0, pending.length - processed);
diff --git a/examples/memory-plugin-shared/lib/pending-queue.mjs b/examples/memory-plugin-shared/lib/pending-queue.mjs
index 6dc45b23d..0666620bc 100644
--- a/examples/memory-plugin-shared/lib/pending-queue.mjs
+++ b/examples/memory-plugin-shared/lib/pending-queue.mjs
@@ -3,9 +3,10 @@
*
* When the OpenViking server is temporarily unreachable, write operations
* (addMessage, commitSession) serialize their payloads to
- * `~/.openviking/pending/` as JSON files. On the next session-start, the
- * queue is replayed in small batches. This is a session-start-triggered retry
- * path with maxRetries/TTL, not a long-running background worker.
+ * `~/.openviking/pending/` as JSON files. The queue is replayed in small
+ * batches, either at session-start (consuming retry budgets, with
+ * maxRetries/TTL) or by a long-running drainer that passes
+ * `consumeRetries: false` so transient failures stay retryable.
*
* Each file contains: { type, sessionId, payload, createdAt, retries, dedupKey }
*
@@ -273,6 +274,23 @@ export async function claimForReplay(filename) {
}
}
+/**
+ * Release a claimed file back to the queue without consuming a retry. The
+ * entry keeps its original filename, retry count, and createdAt position, so a
+ * later run retries it as if this attempt never happened.
+ */
+export async function releaseClaim(claimedFilename) {
+ if (!claimedFilename.endsWith(".processing")) return null;
+ const dir = getPendingDir();
+ const restored = pendingFromProcessingFilename(claimedFilename);
+ try {
+ await rename(join(dir, claimedFilename), join(dir, restored));
+ return restored;
+ } catch {
+ return null;
+ }
+}
+
/**
* Remove a pending entry after successful replay.
*/
@@ -349,9 +367,16 @@ export async function cleanStale() {
*
* @param {Function} fetchJSON - the configured fetchJSON from makeFetchJSON
* @param {Function} log - logger function
+ * @param {object} [options]
+ * @param {boolean} [options.consumeRetries=true] - when false, retryable
+ * failures release their claim instead of incrementing the retry count, so
+ * a background drainer can keep retrying a transient failure without
+ * burning the session-start retry budget. Exhausted and non-retryable
+ * entries are deleted exactly as in the default mode.
* @returns {{ replayed: number, failed: number, skipped: number, deferred: number }}
*/
-export async function replayPending(fetchJSON, log) {
+export async function replayPending(fetchJSON, log, options = {}) {
+ const consumeRetries = options.consumeRetries !== false;
const pending = await listPending();
if (pending.length === 0) {
@@ -426,7 +451,18 @@ export async function replayPending(fetchJSON, log) {
await dequeue(claimedFilename);
skipped++;
} else {
- await incrementRetry(claimedFilename, entry);
+ if (consumeRetries) {
+ await incrementRetry(claimedFilename, entry);
+ } else {
+ const released = await releaseClaim(claimedFilename);
+ log("pending-queue", {
+ action: released ? "replay-deferred" : "release-failed",
+ sessionId: entry.sessionId,
+ type: entry.type,
+ status: res?.result?.status || res?.status,
+ retries: entry.retries || 0,
+ });
+ }
failed++;
if (entry.type === "addMessage") {
deferred += Math.max(0, pending.length - processed);
diff --git a/examples/opencode-plugin/lib/shared/pending-queue.mjs b/examples/opencode-plugin/lib/shared/pending-queue.mjs
index 4b3836a4b..38dccb1c8 100644
--- a/examples/opencode-plugin/lib/shared/pending-queue.mjs
+++ b/examples/opencode-plugin/lib/shared/pending-queue.mjs
@@ -4,9 +4,10 @@
*
* When the OpenViking server is temporarily unreachable, write operations
* (addMessage, commitSession) serialize their payloads to
- * `~/.openviking/pending/` as JSON files. On the next session-start, the
- * queue is replayed in small batches. This is a session-start-triggered retry
- * path with maxRetries/TTL, not a long-running background worker.
+ * `~/.openviking/pending/` as JSON files. The queue is replayed in small
+ * batches, either at session-start (consuming retry budgets, with
+ * maxRetries/TTL) or by a long-running drainer that passes
+ * `consumeRetries: false` so transient failures stay retryable.
*
* Each file contains: { type, sessionId, payload, createdAt, retries, dedupKey }
*
@@ -274,6 +275,23 @@ export async function claimForReplay(filename) {
}
}
+/**
+ * Release a claimed file back to the queue without consuming a retry. The
+ * entry keeps its original filename, retry count, and createdAt position, so a
+ * later run retries it as if this attempt never happened.
+ */
+export async function releaseClaim(claimedFilename) {
+ if (!claimedFilename.endsWith(".processing")) return null;
+ const dir = getPendingDir();
+ const restored = pendingFromProcessingFilename(claimedFilename);
+ try {
+ await rename(join(dir, claimedFilename), join(dir, restored));
+ return restored;
+ } catch {
+ return null;
+ }
+}
+
/**
* Remove a pending entry after successful replay.
*/
@@ -350,9 +368,16 @@ export async function cleanStale() {
*
* @param {Function} fetchJSON - the configured fetchJSON from makeFetchJSON
* @param {Function} log - logger function
+ * @param {object} [options]
+ * @param {boolean} [options.consumeRetries=true] - when false, retryable
+ * failures release their claim instead of incrementing the retry count, so
+ * a background drainer can keep retrying a transient failure without
+ * burning the session-start retry budget. Exhausted and non-retryable
+ * entries are deleted exactly as in the default mode.
* @returns {{ replayed: number, failed: number, skipped: number, deferred: number }}
*/
-export async function replayPending(fetchJSON, log) {
+export async function replayPending(fetchJSON, log, options = {}) {
+ const consumeRetries = options.consumeRetries !== false;
const pending = await listPending();
if (pending.length === 0) {
@@ -427,7 +452,18 @@ export async function replayPending(fetchJSON, log) {
await dequeue(claimedFilename);
skipped++;
} else {
- await incrementRetry(claimedFilename, entry);
+ if (consumeRetries) {
+ await incrementRetry(claimedFilename, entry);
+ } else {
+ const released = await releaseClaim(claimedFilename);
+ log("pending-queue", {
+ action: released ? "replay-deferred" : "release-failed",
+ sessionId: entry.sessionId,
+ type: entry.type,
+ status: res?.result?.status || res?.status,
+ retries: entry.retries || 0,
+ });
+ }
failed++;
if (entry.type === "addMessage") {
deferred += Math.max(0, pending.length - processed);
diff --git a/examples/pi-coding-agent-extension/shared/pending-queue.mjs b/examples/pi-coding-agent-extension/shared/pending-queue.mjs
index 4b3836a4b..38dccb1c8 100644
--- a/examples/pi-coding-agent-extension/shared/pending-queue.mjs
+++ b/examples/pi-coding-agent-extension/shared/pending-queue.mjs
@@ -4,9 +4,10 @@
*
* When the OpenViking server is temporarily unreachable, write operations
* (addMessage, commitSession) serialize their payloads to
- * `~/.openviking/pending/` as JSON files. On the next session-start, the
- * queue is replayed in small batches. This is a session-start-triggered retry
- * path with maxRetries/TTL, not a long-running background worker.
+ * `~/.openviking/pending/` as JSON files. The queue is replayed in small
+ * batches, either at session-start (consuming retry budgets, with
+ * maxRetries/TTL) or by a long-running drainer that passes
+ * `consumeRetries: false` so transient failures stay retryable.
*
* Each file contains: { type, sessionId, payload, createdAt, retries, dedupKey }
*
@@ -274,6 +275,23 @@ export async function claimForReplay(filename) {
}
}
+/**
+ * Release a claimed file back to the queue without consuming a retry. The
+ * entry keeps its original filename, retry count, and createdAt position, so a
+ * later run retries it as if this attempt never happened.
+ */
+export async function releaseClaim(claimedFilename) {
+ if (!claimedFilename.endsWith(".processing")) return null;
+ const dir = getPendingDir();
+ const restored = pendingFromProcessingFilename(claimedFilename);
+ try {
+ await rename(join(dir, claimedFilename), join(dir, restored));
+ return restored;
+ } catch {
+ return null;
+ }
+}
+
/**
* Remove a pending entry after successful replay.
*/
@@ -350,9 +368,16 @@ export async function cleanStale() {
*
* @param {Function} fetchJSON - the configured fetchJSON from makeFetchJSON
* @param {Function} log - logger function
+ * @param {object} [options]
+ * @param {boolean} [options.consumeRetries=true] - when false, retryable
+ * failures release their claim instead of incrementing the retry count, so
+ * a background drainer can keep retrying a transient failure without
+ * burning the session-start retry budget. Exhausted and non-retryable
+ * entries are deleted exactly as in the default mode.
* @returns {{ replayed: number, failed: number, skipped: number, deferred: number }}
*/
-export async function replayPending(fetchJSON, log) {
+export async function replayPending(fetchJSON, log, options = {}) {
+ const consumeRetries = options.consumeRetries !== false;
const pending = await listPending();
if (pending.length === 0) {
@@ -427,7 +452,18 @@ export async function replayPending(fetchJSON, log) {
await dequeue(claimedFilename);
skipped++;
} else {
- await incrementRetry(claimedFilename, entry);
+ if (consumeRetries) {
+ await incrementRetry(claimedFilename, entry);
+ } else {
+ const released = await releaseClaim(claimedFilename);
+ log("pending-queue", {
+ action: released ? "replay-deferred" : "release-failed",
+ sessionId: entry.sessionId,
+ type: entry.type,
+ status: res?.result?.status || res?.status,
+ retries: entry.retries || 0,
+ });
+ }
failed++;
if (entry.type === "addMessage") {
deferred += Math.max(0, pending.length - processed);
diff --git a/examples/zcode-memory-plugin/scripts/shared/pending-queue.mjs b/examples/zcode-memory-plugin/scripts/shared/pending-queue.mjs
index 4b3836a4b..38dccb1c8 100644
--- a/examples/zcode-memory-plugin/scripts/shared/pending-queue.mjs
+++ b/examples/zcode-memory-plugin/scripts/shared/pending-queue.mjs
@@ -4,9 +4,10 @@
*
* When the OpenViking server is temporarily unreachable, write operations
* (addMessage, commitSession) serialize their payloads to
- * `~/.openviking/pending/` as JSON files. On the next session-start, the
- * queue is replayed in small batches. This is a session-start-triggered retry
- * path with maxRetries/TTL, not a long-running background worker.
+ * `~/.openviking/pending/` as JSON files. The queue is replayed in small
+ * batches, either at session-start (consuming retry budgets, with
+ * maxRetries/TTL) or by a long-running drainer that passes
+ * `consumeRetries: false` so transient failures stay retryable.
*
* Each file contains: { type, sessionId, payload, createdAt, retries, dedupKey }
*
@@ -274,6 +275,23 @@ export async function claimForReplay(filename) {
}
}
+/**
+ * Release a claimed file back to the queue without consuming a retry. The
+ * entry keeps its original filename, retry count, and createdAt position, so a
+ * later run retries it as if this attempt never happened.
+ */
+export async function releaseClaim(claimedFilename) {
+ if (!claimedFilename.endsWith(".processing")) return null;
+ const dir = getPendingDir();
+ const restored = pendingFromProcessingFilename(claimedFilename);
+ try {
+ await rename(join(dir, claimedFilename), join(dir, restored));
+ return restored;
+ } catch {
+ return null;
+ }
+}
+
/**
* Remove a pending entry after successful replay.
*/
@@ -350,9 +368,16 @@ export async function cleanStale() {
*
* @param {Function} fetchJSON - the configured fetchJSON from makeFetchJSON
* @param {Function} log - logger function
+ * @param {object} [options]
+ * @param {boolean} [options.consumeRetries=true] - when false, retryable
+ * failures release their claim instead of incrementing the retry count, so
+ * a background drainer can keep retrying a transient failure without
+ * burning the session-start retry budget. Exhausted and non-retryable
+ * entries are deleted exactly as in the default mode.
* @returns {{ replayed: number, failed: number, skipped: number, deferred: number }}
*/
-export async function replayPending(fetchJSON, log) {
+export async function replayPending(fetchJSON, log, options = {}) {
+ const consumeRetries = options.consumeRetries !== false;
const pending = await listPending();
if (pending.length === 0) {
@@ -427,7 +452,18 @@ export async function replayPending(fetchJSON, log) {
await dequeue(claimedFilename);
skipped++;
} else {
- await incrementRetry(claimedFilename, entry);
+ if (consumeRetries) {
+ await incrementRetry(claimedFilename, entry);
+ } else {
+ const released = await releaseClaim(claimedFilename);
+ log("pending-queue", {
+ action: released ? "replay-deferred" : "release-failed",
+ sessionId: entry.sessionId,
+ type: entry.type,
+ status: res?.result?.status || res?.status,
+ retries: entry.retries || 0,
+ });
+ }
failed++;
if (entry.type === "addMessage") {
deferred += Math.max(0, pending.length - processed);