Skip to content

The design language that makes your AI harness better at design.

JavaScript

Fix nightly live-e2e hard-batch flake: pace fake agent writes past Vite's watcher throttle (#1132)

Paul Bakauscf9ec0c37692
diff --git a/tests/live-e2e.test.mjs b/tests/live-e2e.test.mjs
index 7475859ee..29722f5e8 100644
--- a/tests/live-e2e.test.mjs
+++ b/tests/live-e2e.test.mjs
@@ -2038,6 +2038,12 @@ async function runManualEditStage(page, stage, { t, fixture, session, agentMode,
}
if (stage.refreshAfterApply) {
+ // The pre-reload checks above can pass on the overlay's in-page copy, so
+ // a reload is only meaningful once the dev server serves the applied
+ // source. Wait for that explicitly instead of racing its file watcher.
+ await waitForDevServerModules(page, tmp, stage.edits || [], {
+ timeout: agentMode === 'llm' ? 60_000 : 20_000,
+ });
await page.reload({ waitUntil: 'domcontentloaded' });
await waitForHandshake(page);
for (const edit of stage.edits || []) {
@@ -2205,6 +2211,53 @@ function assertSourceMatches(tmp, file, pattern) {
);
}
+/**
+ * Poll the dev server until every source file an Apply stage edited is served
+ * with the applied copy, so a following reload cannot render a module the
+ * server transformed before the last write. Only texts present on disk are
+ * expected; a file the server does not serve under its source path (non-Vite
+ * frameworks) is skipped. A timeout here means the dev server never saw the
+ * final write, which a reload would report as missing copy.
+ */
+async function waitForDevServerModules(page, tmp, edits, { timeout = 20_000 } = {}) {
+ const expectedByFile = new Map();
+ for (const edit of edits) {
+ if (!edit.expectedSourceFile || !edit.expectedVisibleText) continue;
+ const texts = expectedByFile.get(edit.expectedSourceFile) || [];
+ texts.push(edit.expectedVisibleText);
+ expectedByFile.set(edit.expectedSourceFile, texts);
+ }
+ for (const [file, texts] of expectedByFile) {
+ const onDisk = readFileSync(join(tmp, file), 'utf-8');
+ const expected = texts.filter((text) => onDisk.includes(text));
+ if (expected.length === 0) continue;
+ const deadline = Date.now() + timeout;
+ let missing = expected;
+ for (;;) {
+ // Each request is bounded by what is left of the deadline, so a stalled
+ // dev server cannot hold this wait past its timeout.
+ const requestMs = Math.max(250, Math.min(5_000, deadline - Date.now()));
+ const result = await page.evaluate(async ({ url, requestMs }) => {
+ try {
+ const res = await fetch(url, { cache: 'no-store', signal: AbortSignal.timeout(requestMs) });
+ return res.ok ? { text: await res.text() } : { notServed: true };
+ } catch (err) {
+ return { error: String(err?.message || err) };
+ }
+ }, { url: '/' + file, requestMs }).catch((err) => ({ error: err.message }));
+ if (result.notServed) break;
+ if (typeof result.text === 'string') {
+ missing = expected.filter((text) => !result.text.includes(text));
+ if (missing.length === 0) break;
+ }
+ if (Date.now() > deadline) {
+ throw new Error(`dev server still serves a stale /${file} after Apply; missing ${JSON.stringify(missing)} although the file on disk has them`);
+ }
+ await page.waitForTimeout(100);
+ }
+ }
+}
+
async function assertVisibleText(page, selector, text, { timeout = 20_000 } = {}) {
try {
await page.waitForFunction(
diff --git a/tests/live-e2e/agent.mjs b/tests/live-e2e/agent.mjs
index a672e93b1..188c936b4 100644
--- a/tests/live-e2e/agent.mjs
+++ b/tests/live-e2e/agent.mjs
@@ -220,7 +220,11 @@ export function createFakeAgent({ autoRepairMountFailures = true } = {}) {
/** @type {LiveAgent['applyManualEdits']} */
async applyManualEdits(event, context = {}) {
const batch = await loadManualEditEventBatch(event, { tmp: context.tmp });
- return applyManualEditBatchToSource(batch, { tmp: context.tmp, repair: event.repair || null });
+ return applyManualEditBatchToSource(batch, {
+ tmp: context.tmp,
+ repair: event.repair || null,
+ devOrigin: context.devOrigin || null,
+ });
},
/** @type {LiveAgent['handleSteer']} */
@@ -824,7 +828,56 @@ function normalizeVariantSelectorQuotes(css) {
);
}
-export async function applyManualEditBatchToSource(batch, { tmp, sourceEdits, repair = null } = {}) {
+// Vite's bundled chokidar drops a `change` event that arrives within 50 ms of
+// the previous one for the same path (`_throttle(EV_CHANGE, path, 50)` in
+// FSWatcher._emit) and never emits a trailing one. A chunked manual Apply
+// writes the same file once per chunk, and this agent answers each chunk in a
+// few milliseconds, so on Linux (inotify) the last chunk's write was often
+// swallowed: the file on disk was right, but Vite kept serving the previous
+// transform forever and a reload showed old copy (#1081). A real agent spends
+// seconds per chunk and never gets this close.
+//
+// So when the dev server serves the file under its source path (Vite), the
+// fake agent waits until the served module changes after each write. A new
+// transform proves the watcher already emitted that change, so starting the
+// throttle window from the moment it is observed puts the next write to the
+// same file safely outside it. Where the module cannot be fetched, the window
+// is counted from the write itself.
+const WATCHER_CHANGE_THROTTLE_MS = 50;
+const WATCHER_CHANGE_MARGIN_MS = 50;
+const DEV_MODULE_OBSERVE_TIMEOUT_MS = 10_000;
+const lastSourceChangeSeenAt = new Map();
+
+async function writeWatchedSourceFile(fullPath, body, { moduleUrl = null } = {}) {
+ const last = lastSourceChangeSeenAt.get(fullPath);
+ if (last !== undefined) {
+ const wait = last + WATCHER_CHANGE_THROTTLE_MS + WATCHER_CHANGE_MARGIN_MS - Date.now();
+ if (wait > 0) await new Promise((resolve) => setTimeout(resolve, wait));
+ }
+ const previous = await fs.readFile(fullPath, 'utf-8').catch(() => null);
+ const servedBefore = moduleUrl && previous !== body ? await fetchServedModule(moduleUrl) : null;
+ await fs.writeFile(fullPath, body, 'utf-8');
+ if (servedBefore !== null) {
+ const deadline = Date.now() + DEV_MODULE_OBSERVE_TIMEOUT_MS;
+ while (Date.now() < deadline) {
+ const served = await fetchServedModule(moduleUrl);
+ if (served !== null && served !== servedBefore) break;
+ await new Promise((resolve) => setTimeout(resolve, 25));
+ }
+ }
+ lastSourceChangeSeenAt.set(fullPath, Date.now());
+}
+
+async function fetchServedModule(url) {
+ try {
+ const res = await fetch(url, { signal: AbortSignal.timeout(2_000) });
+ return res.ok ? await res.text() : null;
+ } catch {
+ return null;
+ }
+}
+
+export async function applyManualEditBatchToSource(batch, { tmp, sourceEdits, repair = null, devOrigin = null } = {}) {
if (!tmp) throw new Error('manual edit apply requires tmp project root');
const fileCache = new Map();
const filesTouched = new Set();
@@ -948,7 +1001,9 @@ export async function applyManualEditBatchToSource(batch, { tmp, sourceEdits, re
}
for (const file of filesTouched) {
- await fs.writeFile(safeProjectPath(tmp, file), fileCache.get(file), 'utf-8');
+ await writeWatchedSourceFile(safeProjectPath(tmp, file), fileCache.get(file), {
+ moduleUrl: devOrigin ? `${devOrigin}/${file}` : null,
+ });
}
const status = failed.length === 0 ? 'done' : (appliedEntryIds.length > 0 ? 'partial' : 'error');
@@ -1856,6 +1911,7 @@ export async function runAgentLoop({
wrapTarget = { classes: 'hero-title', tag: 'h1' },
steerSourceFile,
steerTarget,
+ devOrigin = null,
}) {
const base = `http://127.0.0.1:${port}`;
// Everything the agent needs to republish a component session without
@@ -2102,7 +2158,7 @@ export async function runAgentLoop({
throw new Error('agent does not implement applyManualEdits');
}
log("Using source hints first; I'll only touch the hinted copy.");
- const result = await agent.applyManualEdits(event, { tmp, engineBin });
+ const result = await agent.applyManualEdits(event, { tmp, engineBin, devOrigin });
if (process.env.IMPECCABLE_E2E_DEBUG) {
log(`manual_edit_apply result: ${JSON.stringify(result)}`);
}
diff --git a/tests/live-e2e/session.mjs b/tests/live-e2e/session.mjs
index be82a7fb3..a399d1e54 100644
--- a/tests/live-e2e/session.mjs
+++ b/tests/live-e2e/session.mjs
@@ -428,6 +428,9 @@ export async function bootFixtureSession({
atomicDelayMs,
steerSourceFile: runtime.steer?.sourceFile,
steerTarget: runtime.steer?.target,
+ // Lets the fake agent see when the dev server has picked up a source
+ // write before it writes the same file again (see agent.mjs).
+ devOrigin: (runtime.scheme || 'http') === 'http' ? `http://127.0.0.1:${devPort}` : null,
};
agentDone = Promise.all([runAgentLoop({ ...loopOptions, log: (m) => log('[worker] ' + m) })]);
}