Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

fix(ok): make the divergence probe's mark-overlap variant overlap (#4516)

Nick Gomez3da0ed1aeab0
diff --git a/packages/desktop/tests/smoke/_helpers/settings-surface.ts b/packages/desktop/tests/smoke/_helpers/settings-surface.ts
index 78393d4f6..5f55b8569 100644
--- a/packages/desktop/tests/smoke/_helpers/settings-surface.ts
+++ b/packages/desktop/tests/smoke/_helpers/settings-surface.ts
@@ -357,12 +357,12 @@ export async function readMenuSnapshotSpellcheck(editor: Page): Promise<boolean
});
}
-export interface EditorSelectionSnapshot {
+interface EditorSelectionSnapshot {
readonly held: boolean;
readonly text: string;
}
-export async function readEditorSelectionSnapshot(editor: Page): Promise<EditorSelectionSnapshot> {
+async function readEditorSelectionSnapshot(editor: Page): Promise<EditorSelectionSnapshot> {
return editorBody(editor).evaluate(async (element) => {
const menu = window.okDesktop?.menu;
if (!menu) throw new Error('the editor window exposes no okDesktop.menu bridge');
@@ -380,6 +380,19 @@ export async function readEditorSelectionSnapshot(editor: Page): Promise<EditorS
});
}
+export async function waitForEditorSelection(
+ editor: Page,
+ expected: string,
+ timeoutMs = 15_000,
+): Promise<void> {
+ await expect
+ .poll(() => readEditorSelectionSnapshot(editor), {
+ timeout: timeoutMs,
+ message: `the editor never held a selection reading ${JSON.stringify(expected)}`,
+ })
+ .toEqual({ held: true, text: expected } satisfies EditorSelectionSnapshot);
+}
+
export async function applicationMenuSpellcheck(
app: ElectronApplication,
action: 'read' | 'click',
diff --git a/packages/desktop/tests/smoke/agent-patch-divergence-probe.e2e.ts b/packages/desktop/tests/smoke/agent-patch-divergence-probe.e2e.ts
index 02723894a..3d6943945 100644
--- a/packages/desktop/tests/smoke/agent-patch-divergence-probe.e2e.ts
+++ b/packages/desktop/tests/smoke/agent-patch-divergence-probe.e2e.ts
@@ -6,6 +6,7 @@ import { setTimeout as wait } from 'node:timers/promises';
import { type ElectronApplication, _electron as electron } from '@playwright/test';
import { desktopLaunchOptions, resolveDesktopTarget } from './_helpers/launch-desktop';
import { PLATFORM_SKIP_REASON, PLATFORM_SUPPORTED, SMOKE_ENABLED } from './_helpers/platform-gate';
+import { waitForEditorSelection } from './_helpers/settings-surface';
import { expect, test } from './_helpers/smoke-test';
const TARGET = resolveDesktopTarget();
@@ -13,21 +14,26 @@ const TARGET = resolveDesktopTarget();
const YDOC_SETTLE_BUDGET_MS = 15_000;
const YDOC_POLL_INTERVAL_MS = 250;
-type Variant = 'same-para' | 'diff-para' | 'mark-overlap' | 'burst' | 'randomized';
+type Variant = 'same-para' | 'diff-para' | 'mark-overlap' | 'burst';
interface ProbeOutcome {
- variant: Variant;
- trials: number;
- httpStatusCodes: number[];
- finalContents: string[];
cherryPresent: boolean[];
bananaAbsent: boolean[];
- humanXCount: number[];
raceFired: boolean[];
+ readFailures: string[][];
+ sawSuccessfulRead: boolean[];
+ lastReadSucceeded: boolean[];
}
const HUMAN_SENTINEL = 'X';
+const HUMAN_TYPED_COUNT = 8;
const AGENT_REPLACE = 'CHERRY';
+const AGENT_FIND = 'BANANA';
+const FIRST_PARAGRAPH = `${AGENT_FIND} is here in the first paragraph.`;
+const SECOND_PARAGRAPH = 'Second paragraph for diff-para variant.';
+const SEED_MARKDOWN = `# Probe\n\n${FIRST_PARAGRAPH}\n\n${SECOND_PARAGRAPH}\n`;
+const SELECTION_SETTLE_MS = 3_000;
+const BOLD_RUN_OVER_REPLACED_FIND = `**${FIRST_PARAGRAPH.replace(AGENT_FIND, AGENT_REPLACE)}${HUMAN_SENTINEL.repeat(HUMAN_TYPED_COUNT)}**`;
interface ApiPort {
port: number;
@@ -41,18 +47,67 @@ async function detectApiPort(page: import('@playwright/test').Page): Promise<Api
return { port: Number(new URL(apiOrigin).port) };
}
-async function fetchYDocContent(port: number, docName: string): Promise<string> {
- const r = await fetch(
- `http://localhost:${port}/api/document?docName=${encodeURIComponent(docName)}`,
- ).catch(() => null);
- if (!r) return '';
- const j = (await r.json().catch(() => ({}))) as { content?: string };
- return j.content ?? '';
+interface YDocRead {
+ content: string;
+ readFailure: string | null;
+}
+
+async function fetchYDocContent(port: number, docName: string): Promise<YDocRead> {
+ let res: Response;
+ try {
+ res = await fetch(
+ `http://localhost:${port}/api/document?docName=${encodeURIComponent(docName)}`,
+ );
+ } catch (err) {
+ const detail = err instanceof Error ? err.message : String(err);
+ return { content: '', readFailure: `fetch rejected: ${detail}` };
+ }
+ const raw = await res.text().catch(() => null);
+ if (raw === null) return { content: '', readFailure: `HTTP ${res.status}: body unreadable` };
+ if (!res.ok) return { content: '', readFailure: `HTTP ${res.status}: ${raw.slice(0, 160)}` };
+ let parsed: unknown;
+ try {
+ parsed = JSON.parse(raw);
+ } catch {
+ return { content: '', readFailure: `unparsable JSON body: ${raw.slice(0, 160)}` };
+ }
+ if (typeof parsed !== 'object' || parsed === null) {
+ return { content: '', readFailure: `non-object JSON body: ${raw.slice(0, 160)}` };
+ }
+ const content = (parsed as { content?: unknown }).content;
+ if (typeof content !== 'string') {
+ return { content: '', readFailure: `no string content field: ${raw.slice(0, 160)}` };
+ }
+ return { content, readFailure: null };
+}
+
+interface SettleState {
+ cherryPresent: boolean;
+ bananaAbsent: boolean;
+ humanXCount: number;
+ settled: boolean;
+}
+
+function readSettleState(variant: Variant, content: string): SettleState {
+ const cherryPresent = content.includes(AGENT_REPLACE);
+ const bananaAbsent = !content.includes(AGENT_FIND);
+ const humanXCount = content.split(HUMAN_SENTINEL).length - 1;
+ const markSettled = variant !== 'mark-overlap' || content.includes(BOLD_RUN_OVER_REPLACED_FIND);
+ return {
+ cherryPresent,
+ bananaAbsent,
+ humanXCount,
+ settled: cherryPresent && bananaAbsent && humanXCount >= 4 && markSettled,
+ };
}
interface RaceResult {
httpStatus: number;
finalContent: string;
+ settled: boolean;
+ readFailures: string[];
+ sawSuccessfulRead: boolean;
+ lastReadSucceeded: boolean;
cherryPresent: boolean;
bananaAbsent: boolean;
humanXCount: number;
@@ -94,6 +149,18 @@ async function seedProbeDocument(port: number, docName: string, markdown: string
}
}
+async function selectWholeParagraph(
+ targetPara: import('@playwright/test').Locator,
+ page: import('@playwright/test').Page,
+ paragraph: string,
+ timeoutMs = 30_000,
+): Promise<void> {
+ await expect(async () => {
+ await targetPara.click({ clickCount: 3 });
+ await waitForEditorSelection(page, paragraph, SELECTION_SETTLE_MS);
+ }).toPass({ timeout: timeoutMs });
+}
+
async function executeRace(opts: {
page: import('@playwright/test').Page;
port: number;
@@ -104,40 +171,37 @@ async function executeRace(opts: {
}): Promise<RaceResult> {
const { page, port, docName, variant, trial, randomizedStaggerMs } = opts;
- const seedContent =
- '# Probe\n\nBANANA is here in the first paragraph.\n\nSecond paragraph for diff-para variant.\n';
+ const seedContent = SEED_MARKDOWN;
expect(seedContent).not.toContain(HUMAN_SENTINEL);
expect(AGENT_REPLACE).not.toContain(HUMAN_SENTINEL);
await seedProbeDocument(port, docName, seedContent);
const editor = page.locator('.ProseMirror[contenteditable="true"]:not(.composer-prosemirror)');
await editor.waitFor({ state: 'visible', timeout: 10_000 });
- await expect(editor).toContainText('BANANA is here', {
+ await expect(editor).toContainText(FIRST_PARAGRAPH, {
timeout: 10_000,
});
let targetPara: import('@playwright/test').Locator;
if (variant === 'diff-para') {
targetPara = page
.locator('.ProseMirror[contenteditable="true"]:not(.composer-prosemirror) p')
- .filter({ hasText: 'Second paragraph' });
+ .filter({ hasText: SECOND_PARAGRAPH });
} else {
targetPara = page
.locator('.ProseMirror[contenteditable="true"]:not(.composer-prosemirror) p')
- .filter({ hasText: 'BANANA' });
+ .filter({ hasText: AGENT_FIND });
}
await targetPara.click();
await page.keyboard.press('End');
if (variant === 'mark-overlap') {
- for (let i = 0; i < 16; i++) {
- await page.keyboard.press('Shift+ArrowLeft');
- }
+ await selectWholeParagraph(targetPara, page, FIRST_PARAGRAPH);
await page.keyboard.press('ControlOrMeta+B');
await page.keyboard.press('End');
await wait(150);
}
- const humanText = HUMAN_SENTINEL.repeat(8);
+ const humanText = HUMAN_SENTINEL.repeat(HUMAN_TYPED_COUNT);
const typingDelay = variant === 'burst' ? 0 : 5;
const agentPatchPromise = (): Promise<Response> =>
@@ -146,7 +210,7 @@ async function executeRace(opts: {
headers: { 'content-type': 'application/json' },
body: JSON.stringify({
docName,
- find: 'BANANA',
+ find: AGENT_FIND,
replace: AGENT_REPLACE,
agentId: trial < 5 ? `probe-${variant}-${trial}` : `probe-${variant}-pool-${trial % 5}`,
agentName: 'probe',
@@ -172,23 +236,39 @@ async function executeRace(opts: {
}
let finalContent = '';
+ const readFailures: string[] = [];
+ let sawSuccessfulRead = false;
+ let lastReadSucceeded = false;
let cherryPresent = false;
let bananaAbsent = false;
let humanXCount = 0;
const deadline = Date.now() + YDOC_SETTLE_BUDGET_MS;
while (Date.now() < deadline) {
- finalContent = await fetchYDocContent(port, docName);
- cherryPresent = finalContent.includes(AGENT_REPLACE);
- bananaAbsent = !finalContent.includes('BANANA');
- humanXCount = finalContent.split(HUMAN_SENTINEL).length - 1;
- if (cherryPresent && bananaAbsent && humanXCount >= 4) break;
+ const read = await fetchYDocContent(port, docName);
+ if (read.readFailure !== null) {
+ readFailures.push(read.readFailure);
+ lastReadSucceeded = false;
+ } else {
+ sawSuccessfulRead = true;
+ lastReadSucceeded = true;
+ finalContent = read.content;
+ const settleState = readSettleState(variant, finalContent);
+ cherryPresent = settleState.cherryPresent;
+ bananaAbsent = settleState.bananaAbsent;
+ humanXCount = settleState.humanXCount;
+ if (settleState.settled) break;
+ }
await wait(YDOC_POLL_INTERVAL_MS);
}
- const raceFired = httpStatus === 200 && !cherryPresent;
+ const raceFired = lastReadSucceeded && httpStatus === 200 && !cherryPresent;
return {
httpStatus,
finalContent,
+ settled: lastReadSucceeded && readSettleState(variant, finalContent).settled,
+ readFailures,
+ sawSuccessfulRead,
+ lastReadSucceeded,
cherryPresent,
bananaAbsent,
humanXCount,
@@ -214,8 +294,7 @@ async function setupElectron(
const contentDir = mkdtempSync(join(tmpdir(), `ok-agent-patch-probe-${variantTag}-`));
const userDataDir = mkdtempSync(join(tmpdir(), `ok-pw-userdata-${variantTag}-`));
const docName = `probe-${variantTag}-${randomUUID().slice(0, 8)}`;
- const initialContent =
- '# Probe\n\nBANANA is here in the first paragraph.\n\nSecond paragraph for diff-para variant.\n';
+ const initialContent = SEED_MARKDOWN;
mkdirSync(join(contentDir, '.ok'), { recursive: true });
writeFileSync(join(contentDir, '.ok', 'config.yml'), 'content:\n dir: .\n');
@@ -249,15 +328,16 @@ async function setupElectron(
await page.waitForLoadState('domcontentloaded');
await expect(
page.locator('.ProseMirror[contenteditable="true"]:not(.composer-prosemirror)'),
- ).toContainText('BANANA', { timeout: 30_000 });
+ ).toContainText(AGENT_FIND, { timeout: 30_000 });
const { port } = await detectApiPort(page);
- const beforeContent = await fetchYDocContent(port, docName);
+ const before = await fetchYDocContent(port, docName);
console.log(
- `[PROBE ${variantTag}] BEFORE — server Y.Doc len=${beforeContent.length}, includes BANANA=${beforeContent.includes('BANANA')}`,
+ `[PROBE ${variantTag}] BEFORE — server Y.Doc len=${before.content.length}, includes ${AGENT_FIND}=${before.content.includes(AGENT_FIND)}, readFailure=${before.readFailure ?? 'none'}`,
);
- expect(beforeContent).toContain('BANANA');
+ expect(before.readFailure).toBeNull();
+ expect(before.content).toContain(AGENT_FIND);
return { app, page, port, docName, contentDir, userDataDir };
}
@@ -278,6 +358,9 @@ test.describe('PRD-6666 — agent-patch divergence (production-built Electron)',
});
console.log('[PROBE A] result:', {
httpStatus: result.httpStatus,
+ readFailures: result.readFailures,
+ sawSuccessfulRead: result.sawSuccessfulRead,
+ lastReadSucceeded: result.lastReadSucceeded,
cherryPresent: result.cherryPresent,
bananaAbsent: result.bananaAbsent,
humanXCount: result.humanXCount,
@@ -286,6 +369,8 @@ test.describe('PRD-6666 — agent-patch divergence (production-built Electron)',
preview: result.finalContent.slice(0, 200),
});
+ expect(result.sawSuccessfulRead).toBe(true);
+ expect(result.lastReadSucceeded).toBe(true);
expect(result.httpStatus).toBe(200);
expect(result.cherryPresent).toBe(true);
expect(result.bananaAbsent).toBe(true);
@@ -308,6 +393,9 @@ test.describe('PRD-6666 — agent-patch divergence (production-built Electron)',
});
console.log('[PROBE B] result:', {
httpStatus: result.httpStatus,
+ readFailures: result.readFailures,
+ sawSuccessfulRead: result.sawSuccessfulRead,
+ lastReadSucceeded: result.lastReadSucceeded,
cherryPresent: result.cherryPresent,
bananaAbsent: result.bananaAbsent,
humanXCount: result.humanXCount,
@@ -316,6 +404,8 @@ test.describe('PRD-6666 — agent-patch divergence (production-built Electron)',
preview: result.finalContent.slice(0, 200),
});
+ expect(result.sawSuccessfulRead).toBe(true);
+ expect(result.lastReadSucceeded).toBe(true);
expect(result.httpStatus).toBe(200);
expect(result.cherryPresent).toBe(true);
expect(result.bananaAbsent).toBe(true);
@@ -338,19 +428,26 @@ test.describe('PRD-6666 — agent-patch divergence (production-built Electron)',
});
console.log('[PROBE C] result:', {
httpStatus: result.httpStatus,
+ readFailures: result.readFailures,
+ sawSuccessfulRead: result.sawSuccessfulRead,
+ lastReadSucceeded: result.lastReadSucceeded,
cherryPresent: result.cherryPresent,
bananaAbsent: result.bananaAbsent,
humanXCount: result.humanXCount,
raceFired: result.raceFired,
finalLen: result.finalContent.length,
preview: result.finalContent.slice(0, 200),
+ expectedBoldRun: BOLD_RUN_OVER_REPLACED_FIND,
});
+ expect(result.sawSuccessfulRead).toBe(true);
+ expect(result.lastReadSucceeded).toBe(true);
expect(result.httpStatus).toBe(200);
expect(result.cherryPresent).toBe(true);
expect(result.bananaAbsent).toBe(true);
expect(result.humanXCount).toBeGreaterThanOrEqual(4);
expect(result.raceFired).toBe(false);
+ expect(result.finalContent).toContain(BOLD_RUN_OVER_REPLACED_FIND);
});
test('Variant D — BURST typing (no keystroke delay) races agent-patch', async ({
@@ -368,6 +465,9 @@ test.describe('PRD-6666 — agent-patch divergence (production-built Electron)',
});
console.log('[PROBE D] result:', {
httpStatus: result.httpStatus,
+ readFailures: result.readFailures,
+ sawSuccessfulRead: result.sawSuccessfulRead,
+ lastReadSucceeded: result.lastReadSucceeded,
cherryPresent: result.cherryPresent,
bananaAbsent: result.bananaAbsent,
humanXCount: result.humanXCount,
@@ -376,6 +476,8 @@ test.describe('PRD-6666 — agent-patch divergence (production-built Electron)',
preview: result.finalContent.slice(0, 200),
});
+ expect(result.sawSuccessfulRead).toBe(true);
+ expect(result.lastReadSucceeded).toBe(true);
expect(result.httpStatus).toBe(200);
expect(result.cherryPresent).toBe(true);
expect(result.bananaAbsent).toBe(true);
@@ -391,14 +493,12 @@ test.describe('PRD-6666 — agent-patch divergence (production-built Electron)',
const TRIALS = process.env.CI ? 25 : 100;
const outcomes: ProbeOutcome = {
- variant: 'randomized',
- trials: TRIALS,
- httpStatusCodes: [],
- finalContents: [],
cherryPresent: [],
bananaAbsent: [],
- humanXCount: [],
raceFired: [],
+ readFailures: [],
+ sawSuccessfulRead: [],
+ lastReadSucceeded: [],
};
for (let trial = 0; trial < TRIALS; trial++) {
const stagger = Math.floor(Math.random() * 10);
@@ -410,12 +510,27 @@ test.describe('PRD-6666 — agent-patch divergence (production-built Electron)',
trial,
randomizedStaggerMs: stagger,
});
- outcomes.httpStatusCodes.push(result.httpStatus);
- outcomes.finalContents.push(result.finalContent);
outcomes.cherryPresent.push(result.cherryPresent);
outcomes.bananaAbsent.push(result.bananaAbsent);
- outcomes.humanXCount.push(result.humanXCount);
outcomes.raceFired.push(result.raceFired);
+ outcomes.readFailures.push(result.readFailures);
+ outcomes.sawSuccessfulRead.push(result.sawSuccessfulRead);
+ outcomes.lastReadSucceeded.push(result.lastReadSucceeded);
+
+ if (!result.settled || !result.sawSuccessfulRead || result.raceFired) {
+ console.log(`[PROBE E trial ${trial}] ANOMALY — stagger=${stagger}ms:`, {
+ httpStatus: result.httpStatus,
+ sawSuccessfulRead: result.sawSuccessfulRead,
+ lastReadSucceeded: result.lastReadSucceeded,
+ raceFired: result.raceFired,
+ cherryPresent: result.cherryPresent,
+ bananaAbsent: result.bananaAbsent,
+ humanXCount: result.humanXCount,
+ readFailures: result.readFailures,
+ finalLen: result.finalContent.length,
+ preview: result.finalContent.slice(0, 200),
+ });
+ }
if (result.raceFired) {
console.log(`[PROBE E trial ${trial}] RACE FIRED — stagger=${stagger}ms:`, {
@@ -429,6 +544,9 @@ test.describe('PRD-6666 — agent-patch divergence (production-built Electron)',
}
}
+ const readFailures = outcomes.readFailures.flat();
+ const trialsWithoutARead = outcomes.sawSuccessfulRead.filter((seen) => !seen).length;
+ const trialsEndingOnAFailedRead = outcomes.lastReadSucceeded.filter((ok) => !ok).length;
const raceCount = outcomes.raceFired.filter(Boolean).length;
const cherryMissedCount = outcomes.cherryPresent.filter((c) => !c).length;
const bananaPresentCount = outcomes.bananaAbsent.filter((a) => !a).length;
@@ -437,8 +555,13 @@ test.describe('PRD-6666 — agent-patch divergence (production-built Electron)',
raceFiredCount: raceCount,
cherryMissedCount,
bananaPresentCount,
+ trialsWithoutARead,
+ trialsEndingOnAFailedRead,
+ readFailures,
});
+ expect(trialsWithoutARead).toBe(0);
+ expect(trialsEndingOnAFailedRead).toBe(0);
expect(raceCount).toBe(0);
expect(cherryMissedCount).toBe(0);
expect(bananaPresentCount).toBe(0);
diff --git a/packages/desktop/tests/smoke/spellcheck-presentation.e2e.ts b/packages/desktop/tests/smoke/spellcheck-presentation.e2e.ts
index d2018d21a..835682443 100644
--- a/packages/desktop/tests/smoke/spellcheck-presentation.e2e.ts
+++ b/packages/desktop/tests/smoke/spellcheck-presentation.e2e.ts
@@ -4,7 +4,6 @@ import { PLATFORM_SKIP_REASON, PLATFORM_SUPPORTED, SMOKE_ENABLED } from './_help
import {
addLanguageInSelector,
closeSettingsDialog,
- type EditorSelectionSnapshot,
editorBody,
findEditorWindow,
launchOnSeededProfile,
@@ -12,11 +11,11 @@ import {
openSettingsDialog,
openSpellingSettings,
pickNonDefaultSelection,
- readEditorSelectionSnapshot,
readSessionSpellingTruth,
seedProjectProfile,
setSpellcheckToggle,
showUserPreferences,
+ waitForEditorSelection,
} from './_helpers/settings-surface';
import { expect, test } from './_helpers/smoke-test';
@@ -63,16 +62,6 @@ async function readStamps(editor: Page): Promise<SurfaceStamps> {
}, STAMP_KEY);
}
-async function waitForEditorSelection(
- editor: Page,
- expected: string,
- timeoutMs = 15_000,
-): Promise<void> {
- await expect
- .poll(() => readEditorSelectionSnapshot(editor), { timeout: timeoutMs })
- .toEqual({ held: true, text: expected } satisfies EditorSelectionSnapshot);
-}
-
async function selectMarkerBackwards(editor: Page, timeoutMs = 30_000): Promise<void> {
await expect(async () => {
await editor.keyboard.press('End');