Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
import { type Span, Telemetry } from '@macro-inc/observability';
import { afterEach, describe, expect, it, vi } from 'vitest';
import * as logger from './logger';
import type { RawUpdate } from './shared';
import { disposeTelemetryFor } from './telemetry';
import { MockLiveSyncSource, MockWALStore, makeTestWAL } from './testing';
import { WAL_TTL_MS, WALSyncer } from './wal';
afterEach(() => {
disposeTelemetryFor('wal-telemetry-test');
vi.restoreAllMocks();
});
function makeWAL(live: MockLiveSyncSource) {
return makeTestWAL(live);
}
async function undeliveredCount(
walStore: MockWALStore<RawUpdate>
): Promise<number> {
const entries = await walStore.getAll();
return entries.filter((e) => !e.delivered).length;
}
// Flush retry logic:
// When append is called while a flush is already running:
// 1. isFlushing = true — the new flush() call returns immediately at the guard
// 2. hasNewPending = true — set by append to mark that something new arrived
// When the current flush finishes:
// - If it succeeded (delivered everything) AND hasNewPending is true → runs flush() again
// to pick up what arrived during the in-flight flush
// - If it failed (network down) → doesn't re-run, leaves items in the store for the
// next reconnect event to trigger
//
// WAL entries are marked delivered (not deleted) on ack. They are only dropped when
// pruneDelivered() is called by the snapshot tick after a durable snapshot save.
describe('WALSyncer', () => {
it('retries failed initialization on demand and shares the retry across callers', async () => {
vi.spyOn(logger, 'logSyncService').mockImplementation(() => {});
const store = new MockWALStore<RawUpdate>();
const error = new DOMException('Temporarily unavailable', 'UnknownError');
const retry = Promise.withResolvers<number>();
const prune = vi
.spyOn(store, 'pruneExpired')
.mockRejectedValueOnce(error)
.mockReturnValueOnce(retry.promise);
const push = vi.fn(async () => true);
const wal = new WALSyncer(store, push);
await expect(wal.ready()).rejects.toBe(error);
expect(prune).toHaveBeenCalledOnce();
const ready = wal.ready();
expect(wal.ready()).toBe(ready);
const append = wal.append(new Uint8Array([1]));
expect(prune).toHaveBeenCalledTimes(2);
expect(await store.count()).toBe(0);
retry.resolve(0);
await append;
await wal.pendingFlush;
await wal.ready();
expect(prune).toHaveBeenCalledTimes(2);
expect(push).toHaveBeenCalledExactlyOnceWith([new Uint8Array([1])]);
});
it('handles constructor setup failure even when no caller immediately awaits ready', async () => {
const log = vi.spyOn(logger, 'logSyncService').mockImplementation(() => {});
const store = new MockWALStore<RawUpdate>();
const error = new Error('Initial prune failed');
const prune = vi.spyOn(store, 'pruneExpired').mockRejectedValueOnce(error);
const wal = new WALSyncer(store, async () => true);
await vi.waitFor(() =>
expect(log).toHaveBeenCalledWith(
expect.objectContaining({
context: {
misc: expect.objectContaining({
errStack: error.stack,
'wal.stage': 'initialization',
}),
},
})
)
);
expect(prune).toHaveBeenCalledOnce();
await wal.ready();
expect(prune).toHaveBeenCalledTimes(2);
});
it('reports distinct flush failures with their stacks during the same outage', async () => {
const log = vi.spyOn(logger, 'logSyncService').mockImplementation(() => {});
const store = new MockWALStore<RawUpdate>();
const wal = new WALSyncer(store, async () => true, 'wal-telemetry-test');
await wal.ready();
const first = new Error('Storage failed');
const second = new Error('Storage failed');
first.stack = 'Error: Storage failed\n at firstOperation';
second.stack = 'Error: Storage failed\n at secondOperation';
const getAll = vi.spyOn(store, 'getAll').mockRejectedValue(first);
await expect(wal.flush()).rejects.toBe(first);
await expect(wal.flush()).rejects.toBe(first);
getAll.mockRejectedValue(second);
await expect(wal.flush()).rejects.toBe(second);
await expect(wal.flush()).rejects.toBe(second);
const errors = log.mock.calls.filter(([entry]) => entry.level === 'error');
expect(errors.map(([entry]) => entry.context.misc?.errStack)).toEqual([
first.stack,
second.stack,
]);
});
it('handles background storage failures once per outage and retries on the next trigger', async () => {
const log = vi.spyOn(logger, 'logSyncService').mockImplementation(() => {});
const store = new MockWALStore<RawUpdate>();
const push = vi.fn(async () => true);
const wal = new WALSyncer(store, push, 'wal-telemetry-test');
await wal.ready();
await store.append(new Uint8Array([1]));
const getAll = vi
.spyOn(store, 'getAll')
.mockRejectedValue(
new DOMException('Storage unavailable', 'UnknownError')
);
void wal.flush();
await vi.waitFor(() =>
expect(
log.mock.calls.filter(([entry]) => entry.level === 'error')
).toHaveLength(1)
);
await expect(wal.flush()).rejects.toMatchObject({ name: 'UnknownError' });
expect(
log.mock.calls.filter(([entry]) => entry.level === 'error')
).toHaveLength(1);
expect(push).not.toHaveBeenCalled();
getAll.mockRestore();
await wal.flush();
expect(push).toHaveBeenCalledExactlyOnceWith([new Uint8Array([1])]);
vi.spyOn(store, 'getAll').mockRejectedValue(
new DOMException('Storage unavailable', 'UnknownError')
);
await expect(wal.flush()).rejects.toMatchObject({ name: 'UnknownError' });
expect(
log.mock.calls.filter(([entry]) => entry.level === 'error')
).toHaveLength(2);
});
it('persists before delivering, marks delivered on ack', async () => {
const live = new MockLiveSyncSource();
const { wal, walStore } = makeWAL(live);
const update = new Uint8Array([1, 2, 3]);
walStore.pause();
await wal.append(update);
expect(await walStore.count()).toBe(1); // written to the WAL
expect(live.pushUpdate).not.toHaveBeenCalled(); // but wal store is paused
walStore.resume();
await wal.pendingFlush;
expect(live.pushUpdate).toHaveBeenCalledExactlyOnceWith([update]);
expect(await walStore.count()).toBe(1); // still there
expect(await undeliveredCount(walStore)).toBe(0); // but marked delivered
});
it('keeps undelivered when live fails', async () => {
const live = new MockLiveSyncSource();
live.setPushResult(false);
const { wal, walStore } = makeWAL(live);
walStore.pause();
await wal.append(new Uint8Array([1, 2, 3]));
walStore.resume();
await wal.pendingFlush;
expect(live.pushUpdate).toHaveBeenCalledOnce();
expect(await undeliveredCount(walStore)).toBe(1);
});
it('batches all pending updates into a single live push', async () => {
const live = new MockLiveSyncSource();
const { wal, walStore } = makeWAL(live);
walStore.pause();
await wal.append(new Uint8Array([1]));
await wal.append(new Uint8Array([2]));
await wal.append(new Uint8Array([3]));
walStore.resume();
await wal.pendingFlush;
expect(live.pushUpdate).toHaveBeenCalledExactlyOnceWith([
new Uint8Array([1]),
new Uint8Array([2]),
new Uint8Array([3]),
]);
expect(await undeliveredCount(walStore)).toBe(0);
});
it('keeps all updates undelivered when the batch push fails', async () => {
const live = new MockLiveSyncSource();
live.setPushResult(false);
const { wal, walStore } = makeWAL(live);
walStore.pause();
await wal.append(new Uint8Array([1]));
await wal.append(new Uint8Array([2]));
await wal.append(new Uint8Array([3]));
walStore.resume();
await wal.pendingFlush;
expect(live.pushUpdate).toHaveBeenCalledOnce();
expect(await undeliveredCount(walStore)).toBe(3);
});
it('retries flush on reconnect', async () => {
const live = new MockLiveSyncSource();
live.setPushResult(false);
const { wal, walStore } = makeWAL(live);
walStore.pause();
await wal.append(new Uint8Array([1, 2, 3]));
walStore.resume();
await wal.pendingFlush;
live.setPushResult(true);
live.emit({
type: 'reconnect',
snapshot: new Uint8Array(),
awareness: new Uint8Array(),
});
await wal.pendingFlush;
expect(await undeliveredCount(walStore)).toBe(0);
});
it('records one delivery outage across repeated failures and ends it on recovery', async () => {
const span: Span = {
span: (() => span) as Span['span'],
run: (operation) => operation(),
setAttr: vi.fn(),
event: vi.fn(),
error: vi.fn(),
traceparent: () => undefined,
injectTraceHeaders: vi.fn(),
end: vi.fn(),
};
const spanSpy = vi.spyOn(Telemetry, 'span').mockReturnValue(span as never);
const store = new MockWALStore<RawUpdate>();
let delivered = false;
const push = vi.fn(async () => delivered);
const wal = new WALSyncer(store, push, 'wal-telemetry-test');
await wal.append(new Uint8Array([1]));
await wal.pendingFlush;
await wal.append(new Uint8Array([2]));
await wal.pendingFlush;
expect(spanSpy).toHaveBeenCalledTimes(1);
expect(spanSpy).toHaveBeenCalledWith('wal.delivery_outage');
expect(span.error).toHaveBeenCalledOnce();
expect(span.end).not.toHaveBeenCalled();
delivered = true;
await wal.flush();
expect(span.event).toHaveBeenCalledWith('wal.ack.confirmed', {
'wal.delivered': 2,
});
expect(span.setAttr).toHaveBeenCalledWith('outcome', 'recovered');
expect(span.end).toHaveBeenCalledOnce();
});
it('flushes updates that arrived during an in-flight flush', async () => {
const live = new MockLiveSyncSource();
const { wal, walStore } = makeWAL(live);
const { resolve } = live.holdNextPush();
await wal.append(new Uint8Array([1]));
await vi.waitFor(() => expect(live.pushUpdate).toHaveBeenCalledOnce());
live.setPushResult(true);
await wal.append(new Uint8Array([2]));
resolve(true);
await wal.pendingFlush;
expect(live.pushUpdate).toHaveBeenCalledTimes(2);
expect(await undeliveredCount(walStore)).toBe(0);
});
it('does not run concurrent flushes', async () => {
const live = new MockLiveSyncSource();
const { wal, walStore } = makeWAL(live);
const { resolve } = live.holdNextPush();
await wal.append(new Uint8Array([1]));
await wal.append(new Uint8Array([2]));
expect(live.pushUpdate).toHaveBeenCalledOnce();
resolve(true);
await wal.pendingFlush;
expect(await undeliveredCount(walStore)).toBe(0);
});
it('does not re-push entries that are already delivered', async () => {
const live = new MockLiveSyncSource();
const { wal, walStore } = makeWAL(live);
await wal.append(new Uint8Array([1]));
await wal.pendingFlush;
expect(await undeliveredCount(walStore)).toBe(0);
// Trigger flush when everything is already delivered — should be a no-op.
const callsBefore = live.pushUpdate.mock.calls.length;
expect(callsBefore).toBe(1); // pushed once during the first flush
await wal.flush();
expect(live.pushUpdate.mock.calls.length).toBe(callsBefore);
});
it('does not push when the WAL is empty', async () => {
const live = new MockLiveSyncSource();
const { wal } = makeWAL(live);
await wal.flush();
expect(live.pushUpdate).not.toHaveBeenCalled();
});
it('pruneDelivered drops delivered entries and keeps undelivered ones', async () => {
const live = new MockLiveSyncSource();
const { wal, walStore } = makeWAL(live);
// First batch: succeeds -> entries 1 and 2 marked delivered.
walStore.pause();
await wal.append(new Uint8Array([1]));
await wal.append(new Uint8Array([2]));
walStore.resume();
await wal.pendingFlush;
// Second batch: fails -> entry 3 stays undelivered.
live.setPushResult(false);
await wal.append(new Uint8Array([3]));
await wal.pendingFlush;
expect(await walStore.count()).toBe(3);
expect(await undeliveredCount(walStore)).toBe(1);
await wal.pruneDelivered();
expect(await walStore.count()).toBe(1); // entry 3 survives
expect(await undeliveredCount(walStore)).toBe(1);
});
it('pruneDelivered does not drop undelivered entries', async () => {
const live = new MockLiveSyncSource();
live.setPushResult(false);
const { wal, walStore } = makeWAL(live);
await wal.append(new Uint8Array([1]));
await wal.pendingFlush;
expect(await undeliveredCount(walStore)).toBe(1);
await wal.pruneDelivered();
expect(await walStore.count()).toBe(1); // undelivered entry survives prune
});
it('pruneExpired drops entries older than the TTL', async () => {
const walStore = new MockWALStore<RawUpdate>();
const now = Date.now();
walStore.seedEntry(new Uint8Array([1]), now - WAL_TTL_MS - 1000); // expired
walStore.seedEntry(new Uint8Array([2]), now - 60_000); // fresh
walStore.seedEntry(new Uint8Array([3]), now - WAL_TTL_MS - 1); // expired
const deleted = await walStore.pruneExpired(WAL_TTL_MS);
expect(deleted).toBe(2);
const remaining = await walStore.getAll();
expect(remaining.map((e) => e.update)).toEqual([new Uint8Array([2])]);
});
it('pruneExpired keeps delivered entries even when expired', async () => {
const walStore = new MockWALStore<RawUpdate>();
const now = Date.now();
walStore.seedEntry(new Uint8Array([1]), now - WAL_TTL_MS - 1000, true); // expired + delivered
walStore.seedEntry(new Uint8Array([2]), now - WAL_TTL_MS - 1000); // expired, undelivered
walStore.seedEntry(new Uint8Array([3]), now - 60_000); // fresh
const deleted = await walStore.pruneExpired(WAL_TTL_MS);
expect(deleted).toBe(1); // only the undelivered expired entry
const remaining = await walStore.getAll();
expect(remaining).toHaveLength(2);
expect(remaining.map((e) => e.update)).toContainEqual(new Uint8Array([1])); // delivered survives
expect(remaining.map((e) => e.update)).toContainEqual(new Uint8Array([3])); // fresh survives
});
it('prunes expired entries on construction so cold-start replay skips them', async () => {
const live = new MockLiveSyncSource();
const walStore = new MockWALStore<RawUpdate>();
walStore.seedEntry(new Uint8Array([1]), Date.now() - WAL_TTL_MS - 1000);
walStore.seedEntry(new Uint8Array([2]), Date.now() - 60_000);
new WALSyncer(walStore, (updates) => live.pushUpdate(updates));
// Constructor schedules the prune asynchronously — wait for it.
await vi.waitFor(async () => {
const entries = await walStore.getAll();
expect(entries).toHaveLength(1);
expect(entries[0]!.update).toEqual(new Uint8Array([2]));
});
});
});