Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(onboarding): resolve the first-run window on accessibility, not pixels

Louis Beaumont01db3332ec62
diff --git a/apps/screenpipe-app-tauri/lib/first-run/learning-window.test.ts b/apps/screenpipe-app-tauri/lib/first-run/learning-window.test.ts
index 824b25b66..6a3d7d89b 100644
--- a/apps/screenpipe-app-tauri/lib/first-run/learning-window.test.ts
+++ b/apps/screenpipe-app-tauri/lib/first-run/learning-window.test.ts
@@ -94,13 +94,16 @@ describe("evidence gate", () => {
).toBe(false);
});
- it("never treats app names as a substitute for captured frames", () => {
+ it("never treats app names as a substitute for observed working time", () => {
// The shipped regression: two apps with one frame each resolved, and
// produced "I watched Google Chrome and Claude … 2 screens indexed".
+ // The floor beside the app count is now active minutes rather than frames,
+ // so this shape must still be refused — it has no observed time at all.
expect(
hasEnoughEvidence(
ok({
total_frames: 2,
+ total_active_minutes: 0,
apps: [
{ name: "Google Chrome", frame_count: 1 },
{ name: "Claude", frame_count: 1 },
@@ -110,6 +113,42 @@ describe("evidence gate", () => {
).toBe(false);
});
+ it("resolves from accessibility alone when screenshots are off", () => {
+ // The bug this gate was changed for. Screenshots disabled means no pixels
+ // and slowly-accruing frames, so the old `frames >= 6 && apps >= 2` clause
+ // could not clear inside the ceiling and the window burned its whole wait
+ // before reporting nothing. Accessibility still names the frontmost app
+ // and capture timestamps still produce active minutes, which is enough to
+ // say something true.
+ expect(
+ hasEnoughEvidence(
+ ok({
+ total_frames: 2,
+ total_active_minutes: 3,
+ apps: [
+ { name: "Google Chrome", frame_count: 1 },
+ { name: "Claude", frame_count: 1 },
+ ],
+ }),
+ ),
+ ).toBe(true);
+ });
+
+ it("resolves on one app plus captured speech", () => {
+ // A call with screenshots off: the screen says almost nothing, the audio
+ // says plenty, and audio is independent of the pixel path.
+ expect(
+ hasEnoughEvidence(
+ ok({
+ total_frames: 1,
+ total_active_minutes: 0,
+ apps: [{ name: "zoom.us", frame_count: 1 }],
+ audio_summary: { segment_count: 12 },
+ }),
+ ),
+ ).toBe(true);
+ });
+
it("resolves on plenty of frames, or fewer frames across several apps", () => {
expect(
hasEnoughEvidence(
diff --git a/apps/screenpipe-app-tauri/lib/first-run/learning-window.ts b/apps/screenpipe-app-tauri/lib/first-run/learning-window.ts
index 61b1d745e..e3aee5ada 100644
--- a/apps/screenpipe-app-tauri/lib/first-run/learning-window.ts
+++ b/apps/screenpipe-app-tauri/lib/first-run/learning-window.ts
@@ -73,11 +73,21 @@ export type FirstRunLearningState = {
const STORAGE_KEY = "screenpipe.first-run.learning-window.v1";
/**
- * Hard ceiling on the wait. Chosen from measurement, not from taste: at 2
- * minutes a warm install had a single app and 2 frames, which is not enough to
- * write a truthful sentence about someone's work.
+ * Hard ceiling on the wait.
+ *
+ * Was 5 minutes, chosen when the evidence gate counted frames and nothing
+ * else. That made the ceiling load-bearing: frames accrue at roughly six a
+ * minute on a normal desktop, so clearing a ten frame floor took most of the
+ * five minutes and the window ran to the ceiling in the common case rather
+ * than the rare one. With `hasEnoughEvidence` now resolving on observed apps,
+ * which accessibility reports immediately and without pixels, the wait is
+ * decided by evidence again and the ceiling goes back to being a backstop.
+ *
+ * Two minutes is the longest a first-run screen may sit before it reads as
+ * broken, and anything the window has not seen by then it is not going to see
+ * by minute five either.
*/
-export const LEARNING_WINDOW_CEILING_MS = 5 * 60 * 1_000;
+export const LEARNING_WINDOW_CEILING_MS = 2 * 60 * 1_000;
/**
* How long after setup a first summary may still be offered.
@@ -107,6 +117,18 @@ export const MIN_EVIDENCE_FRAMES = 10;
export const MIN_MULTI_APP_FRAMES = 6;
export const MIN_EVIDENCE_APPS = 2;
+/**
+ * Observed working time before several apps count as a summary.
+ *
+ * Replaces the frame floor that used to sit beside the app count. Frames are a
+ * pixel-path artifact and vanish as a signal when screenshots are off; active
+ * minutes come from capture timestamps, so accessibility alone populates them.
+ * One minute is the smallest span the summary can describe without the copy
+ * collapsing to "for under a minute", which is what made the thin window read
+ * as broken.
+ */
+export const MIN_EVIDENCE_ACTIVE_MINUTES = 1;
+
/**
* Floor on how early the window may resolve.
*
@@ -115,7 +137,7 @@ export const MIN_EVIDENCE_APPS = 2;
* minute", which makes the whole moment look trivial. Waiting a little buys a
* summary worth reading.
*/
-export const MIN_LEARNING_MS = 90 * 1_000;
+export const MIN_LEARNING_MS = 60 * 1_000;
/** Apps shown in the live readout. More than this reads as noise. */
export const MAX_TRACKED_APPS = 5;
@@ -256,10 +278,35 @@ export function hasEnoughEvidence(activity: ActivitySnapshot): boolean {
if (activity.data_status !== "ok") return false;
const frames = Number(activity.total_frames ?? 0);
const appCount = capturedAppsFrom(activity, 0).length;
+ const audioSegments = Number(activity.audio_summary?.segment_count ?? 0);
+ const activeMinutes = Number(activity.total_active_minutes ?? 0);
+
+ // Several apps over real observed time, with no frame floor beside it.
+ //
+ // The frame count was standing in for two different questions: "did we watch
+ // them long enough" and "was there more than one thing going on". Only the
+ // second needs the app list, and the first is answered directly by active
+ // minutes. Keeping frames in the clause broke the case this window exists to
+ // serve: with screenshots off there are no pixels, frames accrue at roughly
+ // six a minute and carry nothing extra, so `frames >= 6 && apps >= 2` could
+ // sit unmet for the whole ceiling while accessibility had already reported
+ // two apps and a minute of work.
+ //
+ // The active-minutes floor is what keeps the old regression out. Two apps
+ // with one frame each and no elapsed activity is the shape that once
+ // resolved into "I watched Google Chrome and Claude … 2 screens indexed";
+ // it has no observed time, so it still does not resolve here.
+ if (appCount >= MIN_EVIDENCE_APPS && activeMinutes >= MIN_EVIDENCE_ACTIVE_MINUTES) {
+ return true;
+ }
+
+ // One app, but enough captures to show it was sustained rather than a single
+ // redraw as the window opened.
if (frames >= MIN_EVIDENCE_FRAMES) return true;
- // Moving between apps says more per frame than sitting in one, so a couple
- // of apps lowers the bar — but never below a real capture floor.
- return frames >= MIN_MULTI_APP_FRAMES && appCount >= MIN_EVIDENCE_APPS;
+
+ // One app plus captured speech. Audio is independent of the screen, so it is
+ // real evidence on a machine recording a call with screenshots turned off.
+ return appCount >= 1 && audioSegments > 0;
}
export type LearningWindowOpening =
diff --git a/apps/screenpipe-app-tauri/lib/first-run/use-learning-window.test.ts b/apps/screenpipe-app-tauri/lib/first-run/use-learning-window.test.ts
index 5c0b91414..c8afa3a1c 100644
--- a/apps/screenpipe-app-tauri/lib/first-run/use-learning-window.test.ts
+++ b/apps/screenpipe-app-tauri/lib/first-run/use-learning-window.test.ts
@@ -33,6 +33,7 @@ vi.mock("@/lib/first-run/summarize-with-ai", () => ({
import {
LEARNING_WINDOW_CEILING_MS,
LEARNING_WINDOW_GRACE_MS,
+ MIN_LEARNING_MS,
resetLearningWindow,
} from "./learning-window";
import { useLearningWindow } from "./use-learning-window";
@@ -169,8 +170,15 @@ describe("useLearningWindow settings race", () => {
total_frames: 48,
apps: [{ name: "Arc", frame_count: 30 }, { name: "Obsidian", frame_count: 18 }],
} as never);
+ // Must sit above MIN_LEARNING_MS and below LEARNING_WINDOW_CEILING_MS, so
+ // the window opens as `immediate` AND is already old enough to resolve.
+ // Was 3 minutes, which cleared the old 90s floor and the old 5 minute
+ // ceiling; both moved, so this is pinned to the constants rather than to a
+ // literal that silently falls out of the band next time they change.
getOnboardingStatus.mockResolvedValue(
- okStatus(completedAgo(3 * 60_000)) as never,
+ okStatus(
+ completedAgo((MIN_LEARNING_MS + LEARNING_WINDOW_CEILING_MS) / 2),
+ ) as never,
);
const { rerender } = renderHook(