Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

feat(analytics): backfill local Insights history (#2075)

void89ddfc432a81
diff --git a/src/components/InsightsView.tsx b/src/components/InsightsView.tsx
index f50706841..bae228710 100644
--- a/src/components/InsightsView.tsx
+++ b/src/components/InsightsView.tsx
@@ -218,11 +218,13 @@ function MetricCard({
}
function YourUsage({
+ accountId,
dataRetentionEnabled,
isLoaded,
onSyncErrorChange,
syncActive,
}: {
+ accountId: string | null;
dataRetentionEnabled: boolean;
isLoaded: boolean;
onSyncErrorChange: (error: boolean) => void;
@@ -266,7 +268,7 @@ function YourUsage({
if (!(await syncService.syncAnalyticsNow())) {
throw new Error("Insights uploads are not enabled for this account");
}
- const account = await getAccountAnalyticsSummary();
+ const account = await getAccountAnalyticsSummary(accountId);
if (requestId !== requestIdRef.current) return;
setSummary(account);
} else {
@@ -280,7 +282,7 @@ function YourUsage({
} finally {
if (requestId === requestIdRef.current) setLoading(false);
}
- }, [isLoaded, syncActive]);
+ }, [accountId, isLoaded, syncActive]);
useEffect(() => subscribeToAnalyticsRefresh(load, syncActive), [load, syncActive]);
@@ -391,7 +393,7 @@ function YourUsage({
export default function InsightsView({ onSignIn }: InsightsViewProps) {
const { t } = useTranslation();
- const { isLoaded, isSignedIn } = useAuth();
+ const { isLoaded, isSignedIn, user } = useAuth();
const authValidated = hasValidatedAuthContext();
const { dataRetentionEnabled: personalDataRetentionEnabled, insightsSyncEnabled } = useSettings();
const dataRetentionEnabled = usePolicyStore((policyState) =>
@@ -485,6 +487,7 @@ export default function InsightsView({ onSignIn }: InsightsViewProps) {
<TabsContent value="usage" className="mt-6 flex flex-1 flex-col">
<YourUsage
+ accountId={user?.id ?? null}
dataRetentionEnabled={dataRetentionEnabled}
isLoaded={isLoaded}
onSyncErrorChange={setSyncError}
diff --git a/src/helpers/analytics.js b/src/helpers/analytics.js
index 12e3bc0a0..a6820cade 100644
--- a/src/helpers/analytics.js
+++ b/src/helpers/analytics.js
@@ -3,6 +3,13 @@
// via Node's require(esm) with module-syntax detection.
const DAY_MS = 86_400_000;
export const ANALYTICS_ACTIVITY_MONTH_COUNT = 6;
+// Bump only when changed eligibility rules require old transcription rows to
+// be reconsidered. Each local database keeps its own scan cursor per version.
+export const ANALYTICS_HISTORY_BACKFILL_VERSION = 1;
+// Version 0 is deliberately below every exact counting rule. Historical rows
+// reconstructed from legacy desktop storage can therefore be upgraded by the
+// API's richer server history without overwriting a live event.
+export const ANALYTICS_HISTORICAL_COUNTER_VERSION = 0;
export const ANALYTICS_COUNTER_VERSION = 2;
const CJK_SCRIPT_PATTERN =
@@ -77,6 +84,11 @@ function modeFromStoredProvider(provider) {
return "byok";
}
+export function inferHistoricalAnalyticsMode(provider) {
+ const mode = modeFromStoredProvider(provider);
+ return mode === "byok" ? "unknown" : mode;
+}
+
// The provider that actually ran wins when it names a concrete engine or ends
// in "-fallback", which proves the selected route never ran. Streaming provider
// names ("deepgram-streaming") are too coarse to tell BYOK from OpenWhispr
diff --git a/src/helpers/audioManager.js b/src/helpers/audioManager.js
index 7b14a0d6f..9ff3cce1f 100644
--- a/src/helpers/audioManager.js
+++ b/src/helpers/audioManager.js
@@ -1752,6 +1752,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
durationSeconds: this.recordingStartTime
? (Date.now() - this.recordingStartTime) / 1000
: null,
+ analyticsOccurredAt: new Date(this.recordingStartTime || Date.now()).toISOString(),
chunks: this.audioChunks,
segments: this._batchSegments,
mimeType: this.recordingMimeType,
@@ -1772,7 +1773,13 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
this.onStateChange?.({ isRecording: false, isProcessing: false });
}
- persistDiscardedBatchRecording({ durationSeconds, chunks, segments, mimeType }) {
+ persistDiscardedBatchRecording({
+ durationSeconds,
+ analyticsOccurredAt,
+ chunks,
+ segments,
+ mimeType,
+ }) {
// This must run after MediaRecorder's final dataavailable event, so decide
// whether to retain the discarded audio from the snapshot rather than live
// manager state (which may already belong to a new recording).
@@ -1786,7 +1793,8 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
try {
const current = new Blob(chunks, { type: mimeType });
const blob = await this.mergeRecordedSegments([...segments, current]);
- if (blob) await this.saveDiscardedTranscription(blob, durationSeconds);
+ if (blob)
+ await this.saveDiscardedTranscription(blob, durationSeconds, analyticsOccurredAt);
} catch (error) {
const fallback = this.getLargestRecordedSegment([
...segments,
@@ -1794,7 +1802,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
]);
if (fallback) {
try {
- await this.saveDiscardedTranscription(fallback, durationSeconds);
+ await this.saveDiscardedTranscription(fallback, durationSeconds, analyticsOccurredAt);
} catch (fallbackError) {
logger.warn(
"Failed to save discarded recording fallback",
@@ -3930,6 +3938,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
const result = await window.electronAPI.saveTranscription(text, rawText, {
clientTranscriptionId: eventId,
routeKind: this.translationRequested ? "translation" : null,
+ analyticsOccurredAt: occurredAt.toISOString(),
});
if (result?.id) syncService.debouncedPush("transcription", result.id);
@@ -3973,6 +3982,9 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
errorMessage,
errorCode,
routeKind: this.translationRequested ? "translation" : null,
+ ...(metadata?.analyticsOccurredAt
+ ? { analyticsOccurredAt: metadata.analyticsOccurredAt }
+ : {}),
});
if (result?.id) syncService.debouncedPush("transcription", result.id);
@@ -4012,12 +4024,13 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
}
}
- async saveDiscardedTranscription(blob, durationSeconds) {
+ async saveDiscardedTranscription(blob, durationSeconds, analyticsOccurredAt = null) {
let savedId = null;
try {
const result = await window.electronAPI.saveTranscription("", null, {
status: "discarded",
routeKind: this.translationRequested ? "translation" : null,
+ ...(analyticsOccurredAt ? { analyticsOccurredAt } : {}),
});
if (!result?.id) return;
savedId = result.id;
diff --git a/src/helpers/audioStorage.js b/src/helpers/audioStorage.js
index b4dfd5814..3b3b6064d 100644
--- a/src/helpers/audioStorage.js
+++ b/src/helpers/audioStorage.js
@@ -2,6 +2,7 @@ const fs = require("fs");
const path = require("path");
const { app } = require("electron");
const debugLogger = require("./debugLogger");
+const { parseDbTimestamp } = require("./dbTimestamp");
class AudioStorageManager {
constructor() {
@@ -23,8 +24,10 @@ class AudioStorageManager {
_buildFilename(transcriptionId, timestamp) {
if (timestamp) {
- const d = new Date(timestamp);
- if (!isNaN(d.getTime())) {
+ // Named in the user's own wall clock, so the stored instant has to be
+ // resolved before it is read -- a bare SQLite timestamp is UTC, not local.
+ const d = parseDbTimestamp(timestamp);
+ if (d) {
const pad = (n) => String(n).padStart(2, "0");
const date = `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())}`;
const time = `${pad(d.getHours())}-${pad(d.getMinutes())}-${pad(d.getSeconds())}`;
diff --git a/src/helpers/database.js b/src/helpers/database.js
index ebc2f9c4c..2f3433792 100644
--- a/src/helpers/database.js
+++ b/src/helpers/database.js
@@ -6,7 +6,20 @@ const debugLogger = require("./debugLogger");
const { buildNoteSearchQuery } = require("./noteSearch");
const { normalizeStoredSpeakerCount } = require("./speakerCount");
const { parseEventTime } = require("./calendarAvailability");
-const { ANALYTICS_COUNTER_VERSION, summarizeAnalyticsDays } = require("./analytics");
+// An explicit zone marks an instant this app captured at dictation time. A
+// naive timestamp may instead be a sync artifact: upsertTranscriptionFromCloud
+// keeps the cloud created_at but lets timestamp default to the local pull, so
+// a naive value must never outrank created_at when dating a historical row.
+const { hasExplicitTimeZone, parseDbTimestamp, toDbTimestamp } = require("./dbTimestamp");
+const {
+ ANALYTICS_COUNTER_VERSION,
+ ANALYTICS_HISTORY_BACKFILL_VERSION,
+ ANALYTICS_HISTORICAL_COUNTER_VERSION,
+ countSpokenWords,
+ inferHistoricalAnalyticsMode,
+ localDateKey,
+ summarizeAnalyticsDays,
+} = require("./analytics");
const { app } = require("electron");
// Server-enforced trigger cap (openwhispr-api); enforced here so one oversized
@@ -986,6 +999,12 @@ class DatabaseManager {
id INTEGER PRIMARY KEY CHECK (id = 1),
cleared_through TEXT NOT NULL
);
+ CREATE TABLE IF NOT EXISTS analytics_history_backfill_state (
+ id INTEGER PRIMARY KEY CHECK (id = 1),
+ version INTEGER NOT NULL,
+ scanned_through_transcription_id INTEGER NOT NULL DEFAULT 0
+ CHECK (scanned_through_transcription_id >= 0)
+ );
`);
// Repair databases created before analytics deletion tombstones were
// introduced. SQLite has no ADD COLUMN IF NOT EXISTS syntax.
@@ -1263,14 +1282,26 @@ class DatabaseManager {
errorCode = null,
routeKind = null,
clientTranscriptionId = randomUUID(),
+ analyticsOccurredAt = null,
} = {}
) {
try {
if (!this.db) {
throw new Error("Database not initialized");
}
+ // With an occurrence time this column carries when the dictation was
+ // spoken rather than when the row was written -- earlier by the length
+ // of the recording plus transcription. History reads it through
+ // normalizeDbDate, which already branches on a trailing zone.
+ // Keep the existing SQLite-friendly separator so mixed old/new rows
+ // continue to sort chronologically, while the trailing Z marks this as
+ // an exact client-captured instant for clear-state reconciliation.
+ const occurredAt = toDbTimestamp(analyticsOccurredAt);
const stmt = this.db.prepare(
- "INSERT INTO transcriptions (text, raw_text, status, error_message, error_code, route_kind, client_transcription_id) VALUES (?, ?, ?, ?, ?, ?, ?)"
+ `INSERT INTO transcriptions (
+ text, raw_text, status, error_message, error_code, route_kind,
+ client_transcription_id, timestamp
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, COALESCE(?, CURRENT_TIMESTAMP))`
);
const result = stmt.run(
text,
@@ -1279,7 +1310,8 @@ class DatabaseManager {
errorMessage,
errorCode,
routeKind,
- clientTranscriptionId
+ clientTranscriptionId,
+ occurredAt
);
const fetchStmt = this.db.prepare("SELECT * FROM transcriptions WHERE id = ?");
@@ -1292,6 +1324,251 @@ class DatabaseManager {
}
}
+ _ensureAnalyticsHistoryBackfillState(version) {
+ this.db
+ .prepare(
+ `INSERT INTO analytics_history_backfill_state (
+ id, version, scanned_through_transcription_id
+ ) VALUES (1, ?, 0)
+ ON CONFLICT(id) DO UPDATE SET
+ version = excluded.version,
+ scanned_through_transcription_id = 0
+ WHERE analytics_history_backfill_state.version <> excluded.version`
+ )
+ .run(version);
+ return this.db
+ .prepare(
+ `SELECT version, scanned_through_transcription_id
+ FROM analytics_history_backfill_state WHERE id = 1`
+ )
+ .get();
+ }
+
+ getAnalyticsHistoryBackfillState(version = ANALYTICS_HISTORY_BACKFILL_VERSION) {
+ try {
+ if (!this.db) throw new Error("Database not initialized");
+ const safeVersion = Math.max(1, Math.trunc(Number(version)) || 1);
+ return this.db.transaction(() => {
+ const state = this._ensureAnalyticsHistoryBackfillState(safeVersion);
+ const target = this.db
+ .prepare("SELECT COALESCE(MAX(id), 0) AS id FROM transcriptions")
+ .get();
+ return {
+ version: safeVersion,
+ scannedThroughId: Number(state.scanned_through_transcription_id),
+ targetId: Number(target.id),
+ };
+ })();
+ } catch (error) {
+ debugLogger.error(
+ "Error reading analytics history backfill state",
+ { error: error.message },
+ "database"
+ );
+ throw error;
+ }
+ }
+
+ _invalidateAnalyticsHistoryFromTranscription(id) {
+ const resumeBeforeId = Math.max(0, Math.trunc(Number(id)) - 1);
+ this.db
+ .prepare(
+ `INSERT INTO analytics_history_backfill_state (
+ id, version, scanned_through_transcription_id
+ ) VALUES (1, ?, 0)
+ ON CONFLICT(id) DO UPDATE SET
+ version = excluded.version,
+ scanned_through_transcription_id = CASE
+ WHEN analytics_history_backfill_state.version = excluded.version
+ THEN MIN(
+ analytics_history_backfill_state.scanned_through_transcription_id,
+ ?
+ )
+ ELSE 0
+ END`
+ )
+ .run(ANALYTICS_HISTORY_BACKFILL_VERSION, resumeBeforeId);
+ }
+
+ backfillAnalyticsHistoryBatch({
+ afterId = 0,
+ throughId = null,
+ checkpointVersion = null,
+ limit = 250,
+ } = {}) {
+ try {
+ if (!this.db) throw new Error("Database not initialized");
+ const safeLimit = Math.max(1, Math.min(Math.trunc(Number(limit)) || 250, 1_000));
+ const safeAfterId = Math.max(0, Math.trunc(Number(afterId)) || 0);
+ const safeThroughId =
+ throughId === null || throughId === undefined
+ ? null
+ : Math.max(0, Math.trunc(Number(throughId)) || 0);
+ const safeCheckpointVersion =
+ checkpointVersion === null || checkpointVersion === undefined
+ ? null
+ : Math.max(1, Math.trunc(Number(checkpointVersion)) || 1);
+
+ return this.db.transaction(() => {
+ const checkpoint =
+ safeCheckpointVersion === null
+ ? null
+ : this._ensureAnalyticsHistoryBackfillState(safeCheckpointVersion);
+ const effectiveAfterId = checkpoint
+ ? Number(checkpoint.scanned_through_transcription_id)
+ : safeAfterId;
+ if (safeThroughId !== null && effectiveAfterId >= safeThroughId) {
+ return {
+ complete: true,
+ nextCursor: effectiveAfterId,
+ scanned: 0,
+ inserted: 0,
+ skipped: 0,
+ };
+ }
+
+ const clearState = this.db
+ .prepare("SELECT cleared_through FROM analytics_device_clear_state WHERE id = 1")
+ .get();
+ // Legacy SQLite timestamps are completion times without an offset. Once
+ // the user has cleared Insights, only a client-captured occurrence time
+ // can prove that a historical row happened afterward, so an ambiguous
+ // legacy row stays out rather than reviving a cleared counter. That is
+ // the eligibility rule below; the boundary on the instant actually
+ // written is enforced in the loop, where the chosen value is known.
+ const rows = this.db
+ .prepare(
+ `SELECT transcription.id, transcription.client_transcription_id,
+ transcription.text, transcription.raw_text, transcription.timestamp,
+ transcription.created_at,
+ audio_duration_ms, provider, model
+ FROM transcriptions transcription
+ WHERE transcription.id > ?
+ AND (? IS NULL OR transcription.id <= ?)
+ AND transcription.deleted_at IS NULL
+ AND transcription.status = 'completed'
+ AND TRIM(COALESCE(NULLIF(TRIM(transcription.raw_text), ''), transcription.text, '')) != ''
+ AND NOT EXISTS (
+ SELECT 1 FROM analytics_events event
+ WHERE event.event_id = TRIM(transcription.client_transcription_id)
+ )
+ AND (
+ ? IS NULL
+ OR (
+ (TRIM(transcription.timestamp) LIKE '%Z'
+ OR SUBSTR(TRIM(transcription.timestamp), -6, 1) IN ('+', '-'))
+ AND JULIANDAY(transcription.timestamp) > JULIANDAY(?)
+ )
+ )
+ ORDER BY transcription.id ASC
+ LIMIT ?`
+ )
+ .all(
+ effectiveAfterId,
+ safeThroughId,
+ safeThroughId,
+ clearState?.cleared_through ?? null,
+ clearState?.cleared_through ?? null,
+ safeLimit
+ );
+
+ let inserted = 0;
+ let skipped = 0;
+ const clearedThrough = clearState ? Date.parse(clearState.cleared_through) : null;
+ const insert = this.db.prepare(
+ `INSERT INTO analytics_events (
+ event_id, account_id, occurred_at, local_date, word_count,
+ spoken_duration_ms, mode, provider, model, counter_version, created_at
+ ) VALUES (?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, DATETIME(?))
+ ON CONFLICT(event_id) DO NOTHING`
+ );
+ const assignClientId = this.db.prepare(
+ `UPDATE transcriptions SET client_transcription_id = ?
+ WHERE id = ? AND (client_transcription_id IS NULL OR TRIM(client_transcription_id) = '')`
+ );
+
+ for (const row of rows) {
+ const sourceText = row.raw_text?.trim() ? row.raw_text : row.text;
+ const wordCount = countSpokenWords(sourceText);
+ if (wordCount === 0) {
+ skipped += 1;
+ continue;
+ }
+
+ const createdAt = parseDbTimestamp(row.created_at);
+ // A naive timestamp can be a sync artifact rather than an occurrence
+ // time, so it is never the answer: created_at carries the cloud row's
+ // own instant, while timestamp defaulted to the moment of the pull.
+ const occurredAt =
+ (hasExplicitTimeZone(row.timestamp) ? parseDbTimestamp(row.timestamp) : null) ??
+ createdAt;
+ // Guessing a date would put an old dictation on today, inflating
+ // today's counters and manufacturing a current streak out of a row
+ // whose age we could not read. It stays out instead.
+ if (!occurredAt) {
+ skipped += 1;
+ continue;
+ }
+ // Not a restatement of the query's clear filter: that one decides
+ // eligibility from transcription.timestamp, while this guards the
+ // instant actually chosen, which may be created_at. It also catches
+ // what the SQL shape test cannot -- a bare YYYY-MM-DD reads as zoned
+ // there, its day hyphen sitting six characters from the end.
+ if (clearedThrough !== null && occurredAt.getTime() <= clearedThrough) {
+ skipped += 1;
+ continue;
+ }
+
+ const eventId = row.client_transcription_id?.trim() || randomUUID();
+ if (!row.client_transcription_id?.trim()) assignClientId.run(eventId, row.id);
+ const result = insert.run(
+ eventId,
+ occurredAt.toISOString(),
+ localDateKey(occurredAt),
+ wordCount,
+ Number(row.audio_duration_ms) > 0 ? Number(row.audio_duration_ms) : null,
+ inferHistoricalAnalyticsMode(row.provider),
+ row.provider || null,
+ row.model || null,
+ ANALYTICS_HISTORICAL_COUNTER_VERSION,
+ (createdAt ?? occurredAt).toISOString()
+ );
+ if (result.changes > 0) inserted += 1;
+ else skipped += 1;
+ }
+
+ const complete = rows.length < safeLimit;
+ const lastCandidateId =
+ rows.length > 0 ? Number(rows[rows.length - 1].id) : effectiveAfterId;
+ const nextCursor = complete && safeThroughId !== null ? safeThroughId : lastCandidateId;
+ if (checkpoint) {
+ this.db
+ .prepare(
+ `UPDATE analytics_history_backfill_state
+ SET scanned_through_transcription_id = ?
+ WHERE id = 1 AND version = ?`
+ )
+ .run(nextCursor, safeCheckpointVersion);
+ }
+
+ return {
+ complete,
+ nextCursor,
+ scanned: rows.length,
+ inserted,
+ skipped,
+ };
+ })();
+ } catch (error) {
+ debugLogger.error(
+ "Error backfilling analytics history",
+ { error: error.message },
+ "database"
+ );
+ throw error;
+ }
+ }
+
recordAnalyticsEvent({
eventId,
wordCount,
@@ -1402,14 +1679,15 @@ class DatabaseManager {
// The projection is the wire shape: AnalyticsService posts these rows
// verbatim, so every column here has to satisfy the batch endpoint's
// event schema -- occurred_at included, which that schema requires
- // alongside local_date. It also orders the batch, oldest dictation first.
+ // alongside local_date. Exact events go first so rejected historical
+ // rows cannot block current activity during an API rollback.
return this.db
.prepare(
`SELECT event_id, occurred_at, local_date, word_count, spoken_duration_ms,
mode, provider, model, counter_version
FROM analytics_events
WHERE account_id = ? AND sync_status = 'pending' AND deleted_at IS NULL
- ORDER BY occurred_at ASC LIMIT ?`
+ ORDER BY (counter_version = 0) ASC, occurred_at ASC LIMIT ?`
)
.all(accountId, safeLimit);
} catch (error) {
@@ -1772,7 +2050,15 @@ class DatabaseManager {
try {
if (!this.db) throw new Error("Database not initialized");
const stmt = this.db.prepare("UPDATE transcriptions SET text = ?, raw_text = ? WHERE id = ?");
- stmt.run(text, rawText, id);
+ this.db.transaction(() => {
+ const existing = this.db
+ .prepare("SELECT text, raw_text FROM transcriptions WHERE id = ?")
+ .get(id);
+ if (existing && (existing.text !== text || existing.raw_text !== rawText)) {
+ this._invalidateAnalyticsHistoryFromTranscription(id);
+ }
+ stmt.run(text, rawText, id);
+ })();
return { success: true };
} catch (error) {
debugLogger.error("Error updating transcription text", { error: error.message }, "database");
@@ -1786,7 +2072,13 @@ class DatabaseManager {
const stmt = this.db.prepare(
"UPDATE transcriptions SET status = ?, error_message = ?, error_code = ? WHERE id = ?"
);
- stmt.run(status, errorMessage, errorCode, id);
+ this.db.transaction(() => {
+ const existing = this.db.prepare("SELECT status FROM transcriptions WHERE id = ?").get(id);
+ if (existing && existing.status !== status && status === "completed") {
+ this._invalidateAnalyticsHistoryFromTranscription(id);
+ }
+ stmt.run(status, errorMessage, errorCode, id);
+ })();
return { success: true };
} catch (error) {
debugLogger.error(
@@ -6612,9 +6904,24 @@ class DatabaseManager {
upsertTranscriptionFromCloud(cloudTranscription) {
try {
if (!this.db) throw new Error("Database not initialized");
+ const text = cloudTranscription.text ?? "";
+ const rawText = cloudTranscription.raw_text || null;
+ const status = cloudTranscription.status || "completed";
+ // timestamp is what the history list sorts and groups on, so it takes the
+ // cloud row's own instant rather than defaulting to the moment of the
+ // pull -- which would land a whole archive at "now", above everything
+ // spoken since. Deliberately not in the conflict update: a row this
+ // device recorded already carries the recording's start time, which is
+ // more precise than the cloud's creation time for the same dictation.
+ //
+ // The separator is normalized because that sort is a TEXT comparison and
+ // the API sends ISO 8601: "T" (0x54) outranks the space (0x20) every
+ // locally written row uses, so a raw cloud value would sort above every
+ // local dictation from the same UTC day whatever the hour.
+ const cloudOccurredAt = toDbTimestamp(cloudTranscription.created_at);
const stmt = this.db.prepare(`
- INSERT INTO transcriptions (client_transcription_id, cloud_id, text, raw_text, status, sync_status, created_at)
- VALUES (?, ?, ?, ?, ?, 'synced', ?)
+ INSERT INTO transcriptions (client_transcription_id, cloud_id, text, raw_text, status, sync_status, created_at, timestamp)
+ VALUES (?, ?, ?, ?, ?, 'synced', ?, COALESCE(?, CURRENT_TIMESTAMP))
ON CONFLICT(client_transcription_id) DO UPDATE SET
cloud_id = excluded.cloud_id,
text = excluded.text,
@@ -6622,17 +6929,32 @@ class DatabaseManager {
status = excluded.status,
sync_status = 'synced'
`);
- stmt.run(
- cloudTranscription.client_transcription_id,
- cloudTranscription.id,
- cloudTranscription.text ?? "",
- cloudTranscription.raw_text || null,
- cloudTranscription.status || "completed",
- cloudTranscription.created_at
- );
- return this.db
- .prepare("SELECT * FROM transcriptions WHERE client_transcription_id = ?")
- .get(cloudTranscription.client_transcription_id);
+ return this.db.transaction(() => {
+ const existing = this.db
+ .prepare(
+ `SELECT id, text, raw_text, status FROM transcriptions
+ WHERE client_transcription_id = ?`
+ )
+ .get(cloudTranscription.client_transcription_id);
+ if (
+ existing &&
+ (existing.text !== text || existing.raw_text !== rawText || existing.status !== status)
+ ) {
+ this._invalidateAnalyticsHistoryFromTranscription(existing.id);
+ }
+ stmt.run(
+ cloudTranscription.client_transcription_id,
+ cloudTranscription.id,
+ text,
+ rawText,
+ status,
+ cloudTranscription.created_at,
+ cloudOccurredAt
+ );
+ return this.db
+ .prepare("SELECT * FROM transcriptions WHERE client_transcription_id = ?")
+ .get(cloudTranscription.client_transcription_id);
+ })();
} catch (error) {
debugLogger.error(
"Error upserting transcription from cloud",
diff --git a/src/helpers/dbTimestamp.js b/src/helpers/dbTimestamp.js
new file mode 100644
index 000000000..1762358ff
--- /dev/null
+++ b/src/helpers/dbTimestamp.js
@@ -0,0 +1,41 @@
+// SQLite's CURRENT_TIMESTAMP records UTC with no zone designator, and the
+// ECMAScript parser reads that form as *local* time -- so a stored timestamp
+// read back with `new Date` lands hours away from the instant it names. Every
+// main-process reader of a `transcriptions` timestamp goes through here.
+//
+// The renderer has its own copy of this rule in normalizeDbDate
+// (src/utils/dateFormatting.ts); the two must agree, or the same row shows one
+// time in the history list and another in its saved audio's filename.
+
+const EXPLICIT_TIME_ZONE = /(?:Z|[+-]\d{2}:?\d{2})$/i;
+
+function hasExplicitTimeZone(value) {
+ return typeof value === "string" && EXPLICIT_TIME_ZONE.test(value.trim());
+}
+
+/** Returns null for anything unreadable, so callers can fall back explicitly. */
+function parseDbTimestamp(value) {
+ if (typeof value !== "string") return null;
+ const trimmed = value.trim();
+ if (!trimmed) return null;
+ // The stored separator is a space; ISO 8601 wants "T". Normalizing it here
+ // keeps the parse spec-defined rather than relying on the implementation's
+ // legacy date parser, which is free to reject the space-separated form.
+ const isoish = trimmed.replace(" ", "T");
+ const parsed = new Date(EXPLICIT_TIME_ZONE.test(isoish) ? isoish : `${isoish}Z`);
+ return Number.isFinite(parsed.getTime()) ? parsed : null;
+}
+
+/**
+ * The shape a `transcriptions` timestamp is stored in: SQLite's space-separated
+ * form carrying an explicit Z. Both the history list's `ORDER BY timestamp` and
+ * the retention sweep compare that column as TEXT, so every writer has to agree
+ * on the separator -- "T" sorts above a space, so one ISO writer puts its rows
+ * above every other row from the same day. Returns null for anything
+ * unreadable, so callers can fall back to CURRENT_TIMESTAMP.
+ */
+function toDbTimestamp(value) {
+ return parseDbTimestamp(value)?.toISOString().replace("T", " ") ?? null;
+}
+
+module.exports = { hasExplicitTimeZone, parseDbTimestamp, toDbTimestamp };
diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.js
index 666720655..ea5da55ba 100644
--- a/src/helpers/ipcHandlers.js
+++ b/src/helpers/ipcHandlers.js
@@ -4,6 +4,7 @@ const fs = require("fs");
const os = require("os");
const crypto = require("crypto");
const debugLogger = require("./debugLogger");
+const { ANALYTICS_HISTORY_BACKFILL_VERSION } = require("./analytics");
const { PARAKEET_UNSUPPORTED_OS_CODE } = require("./parakeetCapability");
const { getModelType, isSherpaLocalProvider } = require("./parakeetModelInfo");
const { broadcastToWindows } = require("./windowBroadcast");
@@ -654,6 +655,7 @@ class IPCHandlers {
this._retentionSettingsSynced = false;
this._noteFilesEnabled = false;
this._granolaImportPending = null;
+ this._analyticsHistoryBackfillPromise = null;
this.speakerDiarizationEnabled = true;
this.activeMeetingSpeakerConfig = null;
this.whisperVadSettings = {
@@ -703,6 +705,104 @@ class IPCHandlers {
}
}
+ // Reconstructing counters from the transcripts already on disk records exactly
+ // what "keep local history" turns off, so it answers to the same switch the
+ // live path checks in audioManager.saveTranscription. The main process boots
+ // with defaults rather than the user's choice, so an unsynced setting is not
+ // consent either -- the renderer's first sync is what starts this (#1370).
+ _canReconstructAnalyticsHistory() {
+ return this._retentionSettingsSynced && this._retentionSettings.dataRetentionEnabled;
+ }
+
+ /** Whether a signed-in account is bound to this install. */
+ _hasActiveAccountScope() {
+ return Boolean(accountScopeBinding.read());
+ }
+
+ // The switch alone is not enough to start: a managed workspace can force local
+ // history off, and that policy arrives over the network while this scan takes
+ // milliseconds, so the renderer reports the permissive personal default until
+ // it lands. Waiting for the real answer is only possible where there is one --
+ // signed out the policy store stays idle forever and the user's own preference
+ // is the only authority there is. Mid-scan arrival needs no separate check:
+ // a policy that resolves "always_off" flips the switch, which the loop reads.
+ _mayStartAnalyticsHistoryReconstruction() {
+ if (!this._canReconstructAnalyticsHistory()) return false;
+ if (this._retentionSettings.localHistoryPolicyResolved === true) return true;
+ return !this._hasActiveAccountScope();
+ }
+
+ // Reconciliation is best-effort. Analytics reads await it so later-eligible
+ // history shows up before the numbers are read, which means a failure here
+ // must never fail the read itself: a broken scan would otherwise blank an
+ // Insights summary that SQLite could have answered perfectly well.
+ async _ensureAnalyticsHistoryBackfilled() {
+ if (!this._mayStartAnalyticsHistoryReconstruction()) return { inserted: 0, scanned: 0 };
+ if (this._analyticsHistoryBackfillPromise) return this._analyticsHistoryBackfillPromise;
+ // The failure is absorbed inside this promise rather than around the
+ // creator's await, because callers that join an in-flight pass are handed
+ // this promise directly and would otherwise receive the raw rejection --
+ // which is every analytics read that arrives while the startup pass is
+ // still scanning.
+ const backfillPromise = (async () => {
+ let inserted = 0;
+ let scanned = 0;
+ let skipped = 0;
+ let stoppedEarly = false;
+ const state = this.databaseManager.getAnalyticsHistoryBackfillState(
+ ANALYTICS_HISTORY_BACKFILL_VERSION
+ );
+ if (state.scannedThroughId >= state.targetId) return { inserted, scanned };
+ while (true) {
+ // The database reads its persisted cursor again for every batch. An
+ // older row made eligible while this pass yields can move that cursor
+ // backward without being overwritten by stale in-memory progress.
+ const batch = this.databaseManager.backfillAnalyticsHistoryBatch({
+ throughId: state.targetId,
+ checkpointVersion: ANALYTICS_HISTORY_BACKFILL_VERSION,
+ });
+ inserted += batch.inserted;
+ scanned += batch.scanned;
+ skipped += batch.skipped;
+ if (batch.complete) break;
+ await new Promise((resolve) => setImmediate(resolve));
+ // The switch can be turned off while this pass yields -- by the user, or
+ // by a managed policy that resolved after the renderer's first sync sent
+ // the personal default. Re-reading it here stops the scan at the next
+ // batch boundary instead of mining the rest of a history the user has
+ // just opted out of.
+ if (!this._canReconstructAnalyticsHistory()) {
+ stoppedEarly = true;
+ break;
+ }
+ }
+ if (inserted > 0) broadcastToWindows("analytics-changed");
+ if (scanned > 0) {
+ debugLogger.info(
+ stoppedEarly
+ ? "Analytics history backfill stopped: local history was turned off mid-scan"
+ : "Analytics history backfill complete",
+ { inserted, skipped, scanned },
+ "analytics"
+ );
+ }
+ return { inserted, scanned };
+ })().catch((error) => {
+ debugLogger.error("Analytics history backfill failed", { error: error.message }, "analytics");
+ return { inserted: 0, scanned: 0 };
+ });
+ this._analyticsHistoryBackfillPromise = backfillPromise;
+ // Cleared after the assignment above, never inside the pass: a scan that
+ // finishes without ever awaiting would otherwise strand its own resolved
+ // promise here and every later read would join a pass that already ended.
+ void backfillPromise.then(() => {
+ if (this._analyticsHistoryBackfillPromise === backfillPromise) {
+ this._analyticsHistoryBackfillPromise = null;
+ }
+ });
+ return backfillPromise;
+ }
+
// The dictation slot reports its own changes from the renderer. Slots
// registered through IPC have to announce theirs here so macOS can re-derive
// which keys the native Globe listener owns.
@@ -1476,6 +1576,9 @@ class IPCHandlers {
});
ipcMain.handle("analytics-record-event", async (_event, input) => {
+ // The renderer only warns when this write fails, then saves the
+ // transcription as completed anyway -- leaving a row the backfill is
+ // the only thing that will ever reconcile.
const result = this.databaseManager.recordAnalyticsEvent(input);
// Dictation and the control panel are separate renderers, so the
// Insights view can only learn about a new event through the main process.
@@ -1488,11 +1591,13 @@ class IPCHandlers {
});
ipcMain.handle("analytics-get-summary", async () => {
+ await this._ensureAnalyticsHistoryBackfilled();
return this.databaseManager.getAnalyticsSummary();
});
ipcMain.handle("analytics-get-pending", async (_event, limit, context) => {
const accountId = assertAnalyticsSyncContext(context);
+ await this._ensureAnalyticsHistoryBackfilled();
return this.databaseManager.getPendingAnalyticsEvents(limit, accountId);
});
@@ -1523,11 +1628,13 @@ class IPCHandlers {
ipcMain.handle("analytics-count-unclaimed", async (_event, context) => {
assertAnalyticsSyncContext(context);
+ await this._ensureAnalyticsHistoryBackfilled();
return this.databaseManager.countUnclaimedAnalyticsEvents();
});
ipcMain.handle("analytics-count-awaiting-upload", async (_event, context) => {
const accountId = assertAnalyticsSyncContext(context);
+ await this._ensureAnalyticsHistoryBackfilled();
return this.databaseManager.countAnalyticsEventsAwaitingUpload(accountId);
});
@@ -1655,6 +1762,10 @@ class IPCHandlers {
this._retentionSettings = settings;
this._retentionSettingsSynced = true;
this._runRetentionCleanup();
+ // First point at which the local-history switch is known to be real.
+ // After the sweep, so expired transcripts are gone before they can be
+ // reconstructed into counters the sweep would only have to purge.
+ void this._ensureAnalyticsHistoryBackfilled();
},
})
);
@@ -2589,9 +2700,9 @@ class IPCHandlers {
ipcMain.handle("db-get-transcription-by-client-id", (_, clientId) =>
this.databaseManager.getTranscriptionByClientId(clientId)
);
- ipcMain.handle("db-upsert-transcription-from-cloud", (_, cloudTranscription) =>
- this.databaseManager.upsertTranscriptionFromCloud(cloudTranscription)
- );
+ ipcMain.handle("db-upsert-transcription-from-cloud", (_, cloudTranscription) => {
+ return this.databaseManager.upsertTranscriptionFromCloud(cloudTranscription);
+ });
ipcMain.handle("db-mark-transcription-synced", (_, id, cloudId) =>
this.databaseManager.markTranscriptionSynced(id, cloudId)
);
@@ -6391,6 +6502,8 @@ class IPCHandlers {
if (updated) {
setImmediate(() => {
broadcastToWindows("transcription-updated", updated);
+ // A row that just reached "completed" is newly eligible.
+ void this._ensureAnalyticsHistoryBackfilled();
});
}
return { success: true, transcription: updated };
diff --git a/src/helpers/retentionSettings.js b/src/helpers/retentionSettings.js
index 89da52500..992778caf 100644
--- a/src/helpers/retentionSettings.js
+++ b/src/helpers/retentionSettings.js
@@ -4,6 +4,14 @@
const DEFAULT_RETENTION_SETTINGS = {
audioRetentionDays: 30,
transcriptRetentionDays: 0, // 0 = keep transcripts forever
+ // The renderer's policy-aware "keep local history" switch. Defaults to true
+ // so a renderer that predates this field is never read as history-off, and
+ // so the value only becomes trustworthy once hasSynced() says it is real.
+ dataRetentionEnabled: true,
+ // Whether the switch above reflects a settled managed policy rather than the
+ // permissive default the renderer reports while one is still being fetched.
+ // Defaults false: an unreported policy must never read as a resolved one.
+ localHistoryPolicyResolved: false,
};
function toDays(value, fallback) {
@@ -18,10 +26,22 @@ function applyRetentionSettings(current, incoming) {
incoming?.transcriptRetentionDays,
current.transcriptRetentionDays
),
+ dataRetentionEnabled:
+ typeof incoming?.dataRetentionEnabled === "boolean"
+ ? incoming.dataRetentionEnabled
+ : current.dataRetentionEnabled,
+ localHistoryPolicyResolved:
+ typeof incoming?.localHistoryPolicyResolved === "boolean"
+ ? incoming.localHistoryPolicyResolved
+ : current.localHistoryPolicyResolved,
};
const changed =
settings.audioRetentionDays !== current.audioRetentionDays ||
- settings.transcriptRetentionDays !== current.transcriptRetentionDays;
+ settings.transcriptRetentionDays !== current.transcriptRetentionDays ||
+ settings.dataRetentionEnabled !== current.dataRetentionEnabled ||
+ // The policy settling is what unblocks reconstruction, so it has to count
+ // as a change even when every other value stayed put.
+ settings.localHistoryPolicyResolved !== current.localHistoryPolicyResolved;
return { changed, settings };
}
diff --git a/src/hooks/useSettings.ts b/src/hooks/useSettings.ts
index 1bf7f3710..b5bd29b64 100644
--- a/src/hooks/useSettings.ts
+++ b/src/hooks/useSettings.ts
@@ -9,7 +9,11 @@ import type {
SelfHostedType,
} from "../types/electron";
import type { Snippet } from "../utils/snippets";
-import { effectiveAudioRetentionDays } from "../stores/policyRules";
+import {
+ effectiveAudioRetentionDays,
+ effectiveLocalHistoryEnabled,
+ isLocalHistoryPolicyResolved,
+} from "../stores/policyRules";
import { usePolicyStore } from "../stores/policyStore";
export interface TranscriptionSettings {
@@ -185,16 +189,31 @@ function useSettingsInternal() {
}, []);
// Retention periods are enforced by the main process cleanup sweep
- const { audioRetentionDays, transcriptRetentionDays } = store;
+ const { audioRetentionDays, transcriptRetentionDays, dataRetentionEnabled } = store;
const enforcedAudioRetentionDays = usePolicyStore((policyState) =>
effectiveAudioRetentionDays(policyState, audioRetentionDays)
);
+ // Sent alongside the periods because the main process reconstructs Insights
+ // history from stored transcripts, and that must answer to the same switch.
+ const enforcedDataRetentionEnabled = usePolicyStore((policyState) =>
+ effectiveLocalHistoryEnabled(policyState, dataRetentionEnabled)
+ );
+ // Reported alongside the value because history reconstruction reads that
+ // switch as consent, and until the policy settles it is only a default.
+ const localHistoryPolicyResolved = usePolicyStore(isLocalHistoryPolicyResolved);
useEffect(() => {
window.electronAPI?.syncRetentionSettings?.({
audioRetentionDays: enforcedAudioRetentionDays,
transcriptRetentionDays,
+ dataRetentionEnabled: enforcedDataRetentionEnabled,
+ localHistoryPolicyResolved,
});
- }, [enforcedAudioRetentionDays, transcriptRetentionDays]);
+ }, [
+ enforcedAudioRetentionDays,
+ transcriptRetentionDays,
+ enforcedDataRetentionEnabled,
+ localHistoryPolicyResolved,
+ ]);
// Sync startup pre-warming preferences to main process
const {
diff --git a/src/services/AnalyticsService.ts b/src/services/AnalyticsService.ts
index c91590faa..d8da5f32d 100644
--- a/src/services/AnalyticsService.ts
+++ b/src/services/AnalyticsService.ts
@@ -11,10 +11,27 @@ import type {
AnalyticsSyncContext,
PendingAnalyticsEvent,
} from "../types/electron";
+import { ANALYTICS_HISTORICAL_COUNTER_VERSION } from "../helpers/analytics";
const BATCH_SIZE = 200;
+// A pass uploads at most this many batches. Insights waits on a pass before
+// it can read the account summary, so an unbounded drain would hold the view's
+// spinner — and hammer the batch endpoint — for the length of a whole history
+// backfill. What is left over stays pending and still counts as moved work,
+// which keeps the ambient pass cadence tight until the queue is empty.
+const MAX_BATCHES_PER_PASS = 5;
+// How long to leave reconstructed history alone after an API refuses the
+// version it is uploaded at. Short enough that a deploy is picked up within the
+// hour, long enough that a deployment which will never support it costs a
+// couple of dozen requests a day instead of one per pass. A capable answer
+// clears it early, but only a batch that is actually sent can carry one -- on a
+// queue of pure history there is nothing to ride along, so recovery there waits
+// out the window.
+const HISTORICAL_RETRY_COOLDOWN_MS = 60 * 60 * 1000;
+let historicalRetryBlockedUntil = 0;
export const ANALYTICS_SUMMARY_REFRESH_INTERVAL_MS = 5 * 60 * 1000;
export const ANALYTICS_REMOTE_REFRESH_DEBOUNCE_MS = 250;
+const requestedHistoryBackfillAccounts = new Set<string>();
export function subscribeToAnalyticsRefresh(
refresh: () => void | Promise<void>,
@@ -209,7 +226,7 @@ async function runAnalyticsPass({
// batch and one stuck row costs an extra POST per batch behind it.
const offered = new Set<string>();
- while (true) {
+ for (let batch = 0; batch < MAX_BATCHES_PER_PASS; batch += 1) {
// Re-check between batches. Revoking consent while a >200-row drain is in
// flight cannot cancel the active request, but it must stop the next one.
if (!(await canUpload())) return synced;
@@ -221,26 +238,55 @@ async function runAnalyticsPass({
if (fresh.length === 0) return synced;
for (const event of fresh) offered.add(event.event_id);
- // `accepted` is an ack list, not a list of stored rows. The endpoint
- // validates per event and deliberately echoes back the ids it refused as
- // permanently invalid, so marking exactly `accepted` as synced is what
- // retires them. Narrowing this to the ids that were actually stored -- or
- // deriving it from the sibling `rejected` field -- would leave a row that
- // can never validate at the head of the queue forever. A batch the server
- // refuses outright throws and stays pending for the next pass.
+ // An API that predates version zero refuses it identically every time, and
+ // only a deploy can change that answer -- so re-offering those rows on the
+ // ambient pass, on every window focus and after every dictation just repeats
+ // one refusal forever. Back off from history alone: it sorts last, so a
+ // batch that still holds live events is unaffected.
+ const uploadable =
+ Date.now() < historicalRetryBlockedUntil
+ ? fresh.filter((event) => event.counter_version !== ANALYTICS_HISTORICAL_COUNTER_VERSION)
+ : fresh;
+ if (uploadable.length === 0) return synced;
+
+ // `accepted` is an ack list, not a list of stored rows. Only a capable API
+ // can distinguish a permanently invalid version-zero row from an older
+ // deployment rejecting that version altogether. Keep those ids pending
+ // when the capability is absent so an API rollback cannot destroy history.
if (!(await canUpload())) return synced;
- const result = await postToCloud<{ accepted: string[] }>(
- "/api/analytics/events/batch",
- { events: fresh },
- context
- );
+ const result = await postToCloud<{
+ accepted?: string[];
+ rejected?: string[];
+ supportsHistoricalCounterVersion?: boolean;
+ }>("/api/analytics/events/batch", { events: uploadable }, context);
const accepted = Array.isArray(result?.accepted) ? result.accepted : [];
+ const rejected = new Set(Array.isArray(result?.rejected) ? result.rejected : []);
+ const historicalEventIds = new Set(
+ uploadable
+ .filter((event) => event.counter_version === ANALYTICS_HISTORICAL_COUNTER_VERSION)
+ .map((event) => event.event_id)
+ );
+ const acknowledged =
+ result?.supportsHistoricalCounterVersion === true
+ ? accepted
+ : accepted.filter((eventId) => !rejected.has(eventId) || !historicalEventIds.has(eventId));
+ // Any history this answer did not take arms the backoff, not just an
+ // explicit rejection: an older response shape simply omits the rows it
+ // refused. Clearing on the capable answer has to come first, so a capable
+ // API can still retire a genuinely invalid row without arming anything.
+ const acknowledgedIds = new Set(acknowledged);
+ if (result?.supportsHistoricalCounterVersion === true) {
+ historicalRetryBlockedUntil = 0;
+ } else if ([...historicalEventIds].some((eventId) => !acknowledgedIds.has(eventId))) {
+ historicalRetryBlockedUntil = Date.now() + HISTORICAL_RETRY_COOLDOWN_MS;
+ }
- const { updated } = await window.electronAPI.markAnalyticsEventsSynced(accepted, context);
+ const { updated } = await window.electronAPI.markAnalyticsEventsSynced(acknowledged, context);
synced += updated;
// The whole queue fit in one read, so there is nothing behind this batch.
if (events.length < BATCH_SIZE) return synced;
}
+ return synced;
}
const REQUIRED_NONNEGATIVE_SUMMARY_FIELDS = [
@@ -284,26 +330,52 @@ function isAnalyticsSummary(value: unknown): value is AnalyticsSummary {
value.averageWpm === null || isNonnegativeFiniteNumber(value.averageWpm);
const coverageIsValid =
isNonnegativeFiniteNumber(value.wpmCoveragePercent) && value.wpmCoveragePercent <= 100;
+ const retryHintIsValid =
+ value.historyBackfillRetryRequired === undefined ||
+ typeof value.historyBackfillRetryRequired === "boolean";
return (
totalsAreValid &&
averageWpmIsValid &&
coverageIsValid &&
+ retryHintIsValid &&
Array.isArray(value.daily) &&
value.daily.length <= 366 &&
value.daily.every(isAnalyticsDailyBucket)
);
}
-export async function getAccountAnalyticsSummary(): Promise<AnalyticsSummary> {
+export async function getAccountAnalyticsSummary(
+ accountId: string | null = null
+): Promise<AnalyticsSummary> {
+ const requestHistoryBackfill = Boolean(
+ accountId && !requestedHistoryBackfillAccounts.has(accountId)
+ );
+ if (requestHistoryBackfill && accountId) requestedHistoryBackfillAccounts.add(accountId);
const params = new URLSearchParams({
timeZone: Intl.DateTimeFormat().resolvedOptions().timeZone || "UTC",
});
- const summary = await cloudGet<unknown>(`/api/analytics/summary?${params}`);
+ if (requestHistoryBackfill) params.set("backfill", "true");
+
+ let summary: unknown;
+ try {
+ summary = await cloudGet<unknown>(`/api/analytics/summary?${params}`);
+ } catch (error) {
+ // A transient first request must not permanently suppress reconciliation.
+ // The Set is claimed before I/O so overlapping refreshes still collapse to
+ // one trigger for this account and renderer process.
+ if (requestHistoryBackfill && accountId) requestedHistoryBackfillAccounts.delete(accountId);
+ throw error;
+ }
// The cloud is an untrusted JSON boundary. Invalid buckets crash Heatmap
// during render, outside the caller's async fallback, so validate the whole
- // shape before any part of it reaches component state.
+ // shape before any part of it reaches component state. A successful request
+ // already triggered history reconciliation; malformed presentation data
+ // must not start another continuation chain on the next refresh.
if (!isAnalyticsSummary(summary)) {
throw new Error("Malformed analytics summary from cloud");
}
+ if (requestHistoryBackfill && accountId && summary.historyBackfillRetryRequired === true) {
+ requestedHistoryBackfillAccounts.delete(accountId);
+ }
return summary;
}
diff --git a/src/services/LeaderboardService.ts b/src/services/LeaderboardService.ts
index 6088d2191..50f81d305 100644
--- a/src/services/LeaderboardService.ts
+++ b/src/services/LeaderboardService.ts
@@ -265,7 +265,11 @@ async function setParticipation(
): Promise<AnalyticsParticipation> {
const response = await cloudPatchForAuthGeneration<unknown>(
"/api/analytics/participation",
- { enabled },
+ // Joining starts account history reconciliation, so carry the calendar
+ // zone known by this device. Leaving does not start reconciliation.
+ enabled
+ ? { enabled, timeZone: Intl.DateTimeFormat().resolvedOptions().timeZone || "UTC" }
+ : { enabled },
authGeneration
);
const participation = responseData(response, "leaderboard participation");
diff --git a/src/stores/policyRules.ts b/src/stores/policyRules.ts
index 89b9db45a..d92a6e2e0 100644
--- a/src/stores/policyRules.ts
+++ b/src/stores/policyRules.ts
@@ -84,6 +84,27 @@ export function effectiveLocalHistoryEnabled(
return lockedLocalHistoryValue(state) ?? personalPreference;
}
+/**
+ * Whether the managed policy that can force local history off has settled.
+ *
+ * `effectiveLocalHistoryEnabled` resolves an unsettled policy to the user's own
+ * preference, which is the right value to show and to sweep retention with --
+ * but it is a default, not an answer, and one consumer reads that switch as
+ * consent: the main process reconstructs Insights history from stored
+ * transcripts the first time the renderer reports it. A scan finishes in
+ * milliseconds while the policy is a network round trip, so a workspace with
+ * `localHistoryMode: "always_off"` would have its members' existing transcripts
+ * mined before the policy forbidding it ever arrived.
+ *
+ * `idle` is unsettled here even though `isPolicyActionAllowed` treats it as
+ * permissive, because it covers both "no account" and "signed in, fetch not
+ * started". Only the main process can tell those apart, from the account scope
+ * it persists, so it makes that call.
+ */
+export function isLocalHistoryPolicyResolved(state: PolicyDecisionSnapshot): boolean {
+ return state.status === "managed" || state.status === "unmanaged";
+}
+
/** The org-forced local history value, or null when the user may choose. */
export function lockedLocalHistoryValue(state: PolicyDecisionSnapshot): boolean | null {
const mode = managedPolicy(state)?.dataRetention.localHistoryMode;
diff --git a/src/types/electron.ts b/src/types/electron.ts
index 0630d1a8a..6d62a1bf8 100644
--- a/src/types/electron.ts
+++ b/src/types/electron.ts
@@ -212,6 +212,7 @@ export interface AnalyticsSummary {
longestStreakDays: number;
wpmCoveragePercent: number;
daily: AnalyticsDailyBucket[];
+ historyBackfillRetryRequired?: boolean;
}
export type LeaderboardMetric =
@@ -1263,6 +1264,7 @@ declare global {
errorMessage?: string | null;
errorCode?: TranscriptionErrorCode;
clientTranscriptionId?: string;
+ analyticsOccurredAt?: string;
}
) => Promise<{ id: number; success: boolean; transcription?: TranscriptionItem }>;
getTranscriptions: (
@@ -1327,6 +1329,8 @@ declare global {
syncRetentionSettings?: (settings: {
audioRetentionDays: number;
transcriptRetentionDays: number;
+ dataRetentionEnabled: boolean;
+ localHistoryPolicyResolved: boolean;
}) => void;
retryTranscription: (
id: number,
diff --git a/test/helpers/analytics.test.js b/test/helpers/analytics.test.js
index 055ee0684..b32eb1ba4 100644
--- a/test/helpers/analytics.test.js
+++ b/test/helpers/analytics.test.js
@@ -5,6 +5,7 @@ const {
buildAnalyticsActivityDays,
calculateStreaks,
countSpokenWords,
+ inferHistoricalAnalyticsMode,
resolveAnalyticsMode,
summarizeAnalyticsDays,
} = require("../../src/helpers/analytics.js");
@@ -66,6 +67,12 @@ test("analytics credits a fallback to the provider that actually ran", () => {
);
});
+test("historical analytics do not guess an ambiguous provider mode", () => {
+ assert.equal(inferHistoricalAnalyticsMode("local-whisper"), "local");
+ assert.equal(inferHistoricalAnalyticsMode("openwhispr"), "openwhispr_cloud");
+ assert.equal(inferHistoricalAnalyticsMode("deepgram-streaming"), "unknown");
+});
+
test("analytics computes weighted WPM, coverage, and streaks from day totals", () => {
const summary = summarizeAnalyticsDays(
[
diff --git a/test/helpers/analyticsBackfillContract.test.js b/test/helpers/analyticsBackfillContract.test.js
new file mode 100644
index 000000000..708eb3e01
--- /dev/null
+++ b/test/helpers/analyticsBackfillContract.test.js
@@ -0,0 +1,177 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+
+const IPCHandlers = require("../../src/helpers/ipcHandlers");
+
+function completeBatch(overrides = {}) {
+ return {
+ complete: true,
+ nextCursor: 0,
+ scanned: 0,
+ inserted: 0,
+ skipped: 0,
+ ...overrides,
+ };
+}
+
+function createContext(backfillAnalyticsHistoryBatch, targetId = 10) {
+ const state = { scannedThroughId: 0, targetId };
+ return Object.assign(Object.create(IPCHandlers.prototype), {
+ databaseManager: {
+ getAnalyticsHistoryBackfillState: () => ({ version: 1, ...state }),
+ backfillAnalyticsHistoryBatch: (options) => {
+ const result = backfillAnalyticsHistoryBatch({
+ ...options,
+ afterId: state.scannedThroughId,
+ });
+ state.scannedThroughId = result.complete ? state.targetId : result.nextCursor;
+ return result;
+ },
+ },
+ _analyticsHistoryBackfillPromise: null,
+ _retentionSettings: { dataRetentionEnabled: true },
+ _retentionSettingsSynced: true,
+ analyticsHistoryBackfillState: state,
+ });
+}
+
+test("history is not reconstructed until the renderer reports the retention setting", async () => {
+ let scans = 0;
+ const context = createContext(() => {
+ scans += 1;
+ return completeBatch({ nextCursor: 10 });
+ });
+ // The main process boots with defaults, not the user's choice, so before the
+ // first sync it cannot know whether local history is allowed at all.
+ context._retentionSettingsSynced = false;
+
+ await context._ensureAnalyticsHistoryBackfilled();
+
+ assert.equal(scans, 0, "an unsynced retention setting must not be read as consent");
+});
+
+test("history is not reconstructed while local history is turned off", async () => {
+ let scans = 0;
+ const context = createContext(() => {
+ scans += 1;
+ return completeBatch({ nextCursor: 10 });
+ });
+ // The live dictation path skips both the transcript and its counter when this
+ // is off (audioManager.saveTranscription). Mining the transcripts already on
+ // disk for counters would record exactly what the user turned off.
+ context._retentionSettings = { dataRetentionEnabled: false };
+
+ await context._ensureAnalyticsHistoryBackfilled();
+
+ assert.equal(scans, 0, "reconstruction must honour the gate the live path honours");
+});
+
+test("a failed history pass is absorbed and remains retryable", async () => {
+ let attempts = 0;
+ const context = createContext(() => {
+ attempts += 1;
+ if (attempts === 1) throw new Error("broken history row");
+ return completeBatch({ nextCursor: 1 });
+ }, 1);
+
+ assert.deepEqual(await context._ensureAnalyticsHistoryBackfilled(), {
+ inserted: 0,
+ scanned: 0,
+ });
+ assert.deepEqual(await context._ensureAnalyticsHistoryBackfilled(), {
+ inserted: 0,
+ scanned: 0,
+ });
+ assert.equal(attempts, 2);
+});
+
+test("concurrent readers share one pass and completed history stays checkpointed", async () => {
+ let calls = 0;
+ const context = createContext(({ afterId }) => {
+ calls += 1;
+ return afterId === 0
+ ? completeBatch({ complete: false, nextCursor: 5, scanned: 1 })
+ : completeBatch({ nextCursor: 10, scanned: 1 });
+ });
+
+ const first = context._ensureAnalyticsHistoryBackfilled();
+ const joining = context._ensureAnalyticsHistoryBackfilled();
+ assert.deepEqual(await Promise.all([first, joining]), [
+ { inserted: 0, scanned: 2 },
+ { inserted: 0, scanned: 2 },
+ ]);
+ assert.equal(calls, 2, "joining the pass must not start a second scan");
+
+ assert.deepEqual(await context._ensureAnalyticsHistoryBackfilled(), {
+ inserted: 0,
+ scanned: 0,
+ });
+ assert.equal(calls, 2, "a completed pass must short-circuit later reads");
+});
+
+test("history is not reconstructed while an account's policy is still resolving", async () => {
+ let calls = 0;
+ const context = createContext(() => {
+ calls += 1;
+ return completeBatch();
+ });
+ // Signed in, so a workspace could still force local history off; the renderer
+ // has only reported its own permissive default so far.
+ context._hasActiveAccountScope = () => true;
+ context._retentionSettings = { dataRetentionEnabled: true, localHistoryPolicyResolved: false };
+
+ await context._ensureAnalyticsHistoryBackfilled();
+
+ assert.equal(calls, 0, "a default is not consent while a managed policy may still arrive");
+});
+
+test("history is reconstructed for a signed-out user, whose policy never resolves", async () => {
+ let calls = 0;
+ const context = createContext(() => {
+ calls += 1;
+ return completeBatch({ scanned: 1 });
+ });
+ // No account, so the policy store stays idle forever and the user's own
+ // preference is the only authority there is.
+ context._hasActiveAccountScope = () => false;
+ context._retentionSettings = { dataRetentionEnabled: true, localHistoryPolicyResolved: false };
+
+ await context._ensureAnalyticsHistoryBackfilled();
+
+ assert.equal(calls, 1, "an unresolvable policy must not disable the feature");
+});
+
+test("revoking local history stops a pass that is already scanning", async () => {
+ let calls = 0;
+ let context;
+ context = createContext(() => {
+ calls += 1;
+ // The renderer's retention sync lands between the first batch and the
+ // second, which is the only window a yielding scan leaves open.
+ if (calls === 1) context._retentionSettings = { dataRetentionEnabled: false };
+ // A pass that ignores the switch keeps mining until the history runs out.
+ return completeBatch({ complete: calls >= 5, nextCursor: calls, scanned: 1 });
+ });
+
+ await context._ensureAnalyticsHistoryBackfilled();
+
+ assert.equal(calls, 1, "the scan must stop at the next batch boundary, not run to the end");
+});
+
+test("an in-flight pass rereads a durable cursor that moves backward", async () => {
+ let calls = 0;
+ const starts = [];
+ const context = createContext(({ afterId }) => {
+ calls += 1;
+ starts.push(afterId);
+ return afterId < 5
+ ? completeBatch({ complete: false, nextCursor: 5, scanned: 1 })
+ : completeBatch({ nextCursor: 10, scanned: 1 });
+ });
+
+ const inFlight = context._ensureAnalyticsHistoryBackfilled();
+ context.analyticsHistoryBackfillState.scannedThroughId = 0;
+ await inFlight;
+ assert.equal(calls, 3, "the regressed cursor must be revisited before the pass completes");
+ assert.deepEqual(starts, [0, 0, 5]);
+});
diff --git a/test/helpers/analyticsDatabase.test.js b/test/helpers/analyticsDatabase.test.js
index cb8f382b2..838f19780 100644
--- a/test/helpers/analyticsDatabase.test.js
+++ b/test/helpers/analyticsDatabase.test.js
@@ -2,6 +2,26 @@ const test = require("node:test");
const assert = require("node:assert/strict");
const { createDb } = require("./harness/db.js");
+const DatabaseManager = require("../../src/helpers/database.js");
+const {
+ ANALYTICS_HISTORY_BACKFILL_VERSION,
+ localDateKey,
+} = require("../../src/helpers/analytics.js");
+
+function reconcileThroughCheckpoint(db, limit = 250) {
+ const state = db.getAnalyticsHistoryBackfillState(ANALYTICS_HISTORY_BACKFILL_VERSION);
+ const batches = [];
+ do {
+ batches.push(
+ db.backfillAnalyticsHistoryBatch({
+ throughId: state.targetId,
+ checkpointVersion: ANALYTICS_HISTORY_BACKFILL_VERSION,
+ limit,
+ })
+ );
+ } while (!batches[batches.length - 1].complete);
+ return batches;
+}
function recordEvent(db, eventId, overrides = {}) {
db.recordAnalyticsEvent({
@@ -17,6 +37,567 @@ function recordEvent(db, eventId, overrides = {}) {
});
}
+test("historical transcriptions reconcile in restart-safe account-neutral batches", (t) => {
+ const db = createDb(t);
+ if (!db) return;
+
+ const insertTranscription = db.db.prepare(
+ `INSERT INTO transcriptions (
+ text, raw_text, status, client_transcription_id, timestamp, created_at,
+ audio_duration_ms, provider, model, deleted_at
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
+ );
+ insertTranscription.run(
+ "enhanced text",
+ "one two three",
+ "completed",
+ "legacy-local",
+ "2026-09-01T09:59:00.000Z",
+ "2026-09-01 10:00:00",
+ 2_000,
+ "local-whisper",
+ "small",
+ null
+ );
+ insertTranscription.run(
+ "four five",
+ null,
+ "completed",
+ "legacy-ambiguous",
+ "2026-09-02T09:59:00.000Z",
+ "2026-09-02 10:00:00",
+ null,
+ "deepgram-streaming",
+ "nova-3",
+ null
+ );
+ insertTranscription.run(
+ "ambiguous post-clear words",
+ null,
+ "completed",
+ "legacy-post-clear-ambiguous",
+ "2026-09-03 10:00:00",
+ "2026-09-03 10:00:00",
+ null,
+ null,
+ null,
+ null
+ );
+ insertTranscription.run(
+ "cleared words",
+ null,
+ "completed",
+ "legacy-cleared",
+ "2026-08-01T09:59:00.000Z",
+ "2026-08-01 10:00:00",
+ null,
+ null,
+ null,
+ null
+ );
+ insertTranscription.run(
+ "failed words",
+ null,
+ "failed",
+ "legacy-failed",
+ "2026-09-03 10:00:00",
+ "2026-09-03 10:00:00",
+ null,
+ null,
+ null,
+ null
+ );
+ insertTranscription.run(
+ "timestamp unavailable",
+ null,
+ "completed",
+ "legacy-invalid-time",
+ "not-a-time",
+ "not-a-time",
+ null,
+ null,
+ null,
+ null
+ );
+ insertTranscription.run(
+ "deleted words",
+ null,
+ "completed",
+ "legacy-deleted",
+ "2026-09-03 10:00:00",
+ "2026-09-03 10:00:00",
+ null,
+ null,
+ null,
+ "2026-09-03 11:00:00"
+ );
+ insertTranscription.run(
+ "existing words",
+ null,
+ "completed",
+ "legacy-existing",
+ "2026-09-04 10:00:00",
+ "2026-09-04 10:00:00",
+ null,
+ null,
+ null,
+ null
+ );
+ db.db
+ .prepare(
+ "INSERT INTO analytics_device_clear_state (id, cleared_through) VALUES (1, '2026-08-15T00:00:00.000Z')"
+ )
+ .run();
+ recordEvent(db, "legacy-existing", {
+ wordCount: 2,
+ occurredAt: "2026-09-04T10:00:00.000Z",
+ });
+ db.db
+ .prepare(
+ "UPDATE analytics_events SET deleted_at = '2026-09-04T11:00:00.000Z' WHERE event_id = 'legacy-existing'"
+ )
+ .run();
+ db.setActiveAccountId("account-a");
+
+ const batches = [];
+ let afterId = 0;
+ do {
+ const batch = db.backfillAnalyticsHistoryBatch({ afterId, limit: 1 });
+ batches.push(batch);
+ afterId = batch.nextCursor;
+ } while (!batches[batches.length - 1].complete);
+
+ assert.equal(
+ batches.reduce((total, batch) => total + batch.inserted, 0),
+ 2
+ );
+ assert.equal(db.getAnalyticsSummary().totalWords, 5);
+ assert.equal(db.getAnalyticsSummary().totalDictations, 2);
+ assert.equal(db.countUnclaimedAnalyticsEvents(), 2, "the active account does not adopt history");
+ assert.deepEqual(
+ db.db
+ .prepare(
+ `SELECT event_id, account_id, word_count, spoken_duration_ms, mode,
+ counter_version, created_at
+ FROM analytics_events WHERE deleted_at IS NULL ORDER BY event_id`
+ )
+ .all(),
+ [
+ {
+ event_id: "legacy-ambiguous",
+ account_id: null,
+ word_count: 2,
+ spoken_duration_ms: null,
+ mode: "unknown",
+ counter_version: 0,
+ created_at: "2026-09-02 10:00:00",
+ },
+ {
+ event_id: "legacy-local",
+ account_id: null,
+ word_count: 3,
+ spoken_duration_ms: 2_000,
+ mode: "local",
+ counter_version: 0,
+ created_at: "2026-09-01 10:00:00",
+ },
+ ]
+ );
+ assert.deepEqual(db.backfillAnalyticsHistoryBatch(), {
+ complete: true,
+ nextCursor: 0,
+ scanned: 0,
+ inserted: 0,
+ skipped: 0,
+ });
+});
+
+test("analytics reconciliation picks up later eligibility and usable processed text", (t) => {
+ const db = createDb(t);
+ if (!db) return;
+
+ const insert = db.db.prepare(
+ `INSERT INTO transcriptions (
+ text, raw_text, status, client_transcription_id, timestamp, created_at
+ ) VALUES (?, ?, ?, ?, ?, ?)`
+ );
+ insert.run(
+ "processed text wins",
+ " ",
+ "completed",
+ "empty-raw",
+ "2026-09-01 10:00:00",
+ "2026-09-01 10:00:00"
+ );
+ insert.run(
+ "retry succeeds later",
+ null,
+ "failed",
+ "retried-later",
+ "2026-09-02 10:00:00",
+ "2026-09-02 10:00:00"
+ );
+
+ assert.equal(db.backfillAnalyticsHistoryBatch().inserted, 1);
+ assert.equal(db.getAnalyticsSummary().totalWords, 3);
+
+ db.db
+ .prepare("UPDATE transcriptions SET status = 'completed' WHERE client_transcription_id = ?")
+ .run("retried-later");
+ assert.equal(db.backfillAnalyticsHistoryBatch().inserted, 1);
+ assert.equal(db.getAnalyticsSummary().totalWords, 6);
+
+ insert.run(
+ "pulled after startup",
+ null,
+ "completed",
+ "pulled-later",
+ "2025-01-01 10:00:00",
+ "2025-01-01 10:00:00"
+ );
+ assert.equal(db.backfillAnalyticsHistoryBatch().inserted, 1);
+ assert.equal(db.backfillAnalyticsHistoryBatch().scanned, 0);
+});
+
+test("analytics history checkpoint persists progress and scans only the new tail", (t) => {
+ const db = createDb(t);
+ if (!db) return;
+
+ const insert = db.db.prepare(
+ `INSERT INTO transcriptions (
+ text, status, client_transcription_id, timestamp, created_at
+ ) VALUES (?, 'completed', ?, ?, ?)`
+ );
+ insert.run("one two", "checkpoint-1", "2026-09-01T10:00:00.000Z", "2026-09-01 10:00:00");
+ insert.run("three four", "checkpoint-2", "2026-09-02T10:00:00.000Z", "2026-09-02 10:00:00");
+
+ assert.deepEqual(db.getAnalyticsHistoryBackfillState(), {
+ version: ANALYTICS_HISTORY_BACKFILL_VERSION,
+ scannedThroughId: 0,
+ targetId: 2,
+ });
+ assert.equal(
+ reconcileThroughCheckpoint(db, 1).reduce((total, batch) => total + batch.inserted, 0),
+ 2
+ );
+ assert.deepEqual(db.getAnalyticsHistoryBackfillState(), {
+ version: ANALYTICS_HISTORY_BACKFILL_VERSION,
+ scannedThroughId: 2,
+ targetId: 2,
+ });
+ assert.deepEqual(reconcileThroughCheckpoint(db), [
+ {
+ complete: true,
+ nextCursor: 2,
+ scanned: 0,
+ inserted: 0,
+ skipped: 0,
+ },
+ ]);
+
+ insert.run("five six", "checkpoint-3", "2026-09-03T10:00:00.000Z", "2026-09-03 10:00:00");
+ assert.deepEqual(db.getAnalyticsHistoryBackfillState(), {
+ version: ANALYTICS_HISTORY_BACKFILL_VERSION,
+ scannedThroughId: 2,
+ targetId: 3,
+ });
+ assert.equal(reconcileThroughCheckpoint(db)[0].inserted, 1);
+ assert.equal(db.getAnalyticsSummary().totalDictations, 3);
+
+ assert.deepEqual(db.getAnalyticsHistoryBackfillState(ANALYTICS_HISTORY_BACKFILL_VERSION + 1), {
+ version: ANALYTICS_HISTORY_BACKFILL_VERSION + 1,
+ scannedThroughId: 0,
+ targetId: 3,
+ });
+});
+
+test("a reopened database reuses its completed analytics history checkpoint", (t) => {
+ const db = createDb(t);
+ if (!db) return;
+
+ db.saveTranscription("persist this checkpoint", null, {
+ clientTranscriptionId: "checkpoint-reopen",
+ analyticsOccurredAt: "2026-09-01T10:00:00.000Z",
+ });
+ reconcileThroughCheckpoint(db);
+ db.db.close();
+
+ const reopened = new DatabaseManager();
+ try {
+ assert.deepEqual(reopened.getAnalyticsHistoryBackfillState(), {
+ version: ANALYTICS_HISTORY_BACKFILL_VERSION,
+ scannedThroughId: 1,
+ targetId: 1,
+ });
+ assert.equal(reconcileThroughCheckpoint(reopened)[0].scanned, 0);
+ } finally {
+ reopened.db.close();
+ }
+});
+
+test("schema setup adds the checkpoint table to an existing database without a migration", (t) => {
+ const db = createDb(t);
+ if (!db) return;
+
+ db.db.exec("DROP TABLE analytics_history_backfill_state");
+ db.db.close();
+
+ const reopened = new DatabaseManager();
+ try {
+ assert.deepEqual(reopened.getAnalyticsHistoryBackfillState(), {
+ version: ANALYTICS_HISTORY_BACKFILL_VERSION,
+ scannedThroughId: 0,
+ targetId: 0,
+ });
+ } finally {
+ reopened.db.close();
+ }
+});
+
+test("older eligibility changes rewind the durable analytics checkpoint", (t) => {
+ const db = createDb(t);
+ if (!db) return;
+
+ const failed = db.db
+ .prepare(
+ `INSERT INTO transcriptions (
+ text, status, client_transcription_id, timestamp, created_at
+ ) VALUES ('retry these words', 'failed', 'checkpoint-retry',
+ '2026-09-01T10:00:00.000Z', '2026-09-01 10:00:00')`
+ )
+ .run();
+ db.db
+ .prepare(
+ `INSERT INTO transcriptions (
+ text, status, client_transcription_id, timestamp, created_at
+ ) VALUES ('already completed', 'completed', 'checkpoint-complete',
+ '2026-09-02T10:00:00.000Z', '2026-09-02 10:00:00')`
+ )
+ .run();
+ const empty = db.db
+ .prepare(
+ `INSERT INTO transcriptions (
+ text, status, client_transcription_id, timestamp, created_at
+ ) VALUES ('', 'completed', 'checkpoint-empty',
+ '2026-09-03T10:00:00.000Z', '2026-09-03 10:00:00')`
+ )
+ .run();
+ reconcileThroughCheckpoint(db);
+ assert.equal(db.getAnalyticsHistoryBackfillState().scannedThroughId, 3);
+
+ db.updateTranscriptionStatus(Number(failed.lastInsertRowid), "completed");
+ assert.equal(db.getAnalyticsHistoryBackfillState().scannedThroughId, 0);
+ assert.equal(reconcileThroughCheckpoint(db)[0].inserted, 1);
+
+ const unchanged = db.getTranscriptionById(Number(failed.lastInsertRowid));
+ db.upsertTranscriptionFromCloud({
+ id: "cloud-retry",
+ client_transcription_id: unchanged.client_transcription_id,
+ text: unchanged.text,
+ raw_text: unchanged.raw_text,
+ status: unchanged.status,
+ created_at: unchanged.created_at,
+ });
+ assert.equal(
+ db.getAnalyticsHistoryBackfillState().scannedThroughId,
+ 3,
+ "an unchanged cloud replay must not rewind the cursor"
+ );
+
+ db.upsertTranscriptionFromCloud({
+ id: "cloud-retry",
+ client_transcription_id: unchanged.client_transcription_id,
+ text: "changed cloud text",
+ raw_text: unchanged.raw_text,
+ status: unchanged.status,
+ created_at: unchanged.created_at,
+ });
+ assert.equal(db.getAnalyticsHistoryBackfillState().scannedThroughId, 0);
+ reconcileThroughCheckpoint(db);
+
+ db.updateTranscriptionText(Number(empty.lastInsertRowid), "now usable", null);
+ assert.equal(db.getAnalyticsHistoryBackfillState().scannedThroughId, 2);
+ assert.equal(reconcileThroughCheckpoint(db)[0].inserted, 1);
+});
+
+test("analytics history checkpoint advances atomically with each batch", (t) => {
+ const db = createDb(t);
+ if (!db) return;
+
+ db.db
+ .prepare(
+ `INSERT INTO transcriptions (
+ text, status, client_transcription_id, timestamp, created_at
+ ) VALUES
+ ('first row', 'completed', 'atomic-1',
+ '2026-09-01T10:00:00.000Z', '2026-09-01 10:00:00'),
+ ('second row', 'completed', 'atomic-2',
+ '2026-09-02T10:00:00.000Z', '2026-09-02 10:00:00')`
+ )
+ .run();
+ const state = db.getAnalyticsHistoryBackfillState();
+ const first = db.backfillAnalyticsHistoryBatch({
+ throughId: state.targetId,
+ checkpointVersion: ANALYTICS_HISTORY_BACKFILL_VERSION,
+ limit: 1,
+ });
+ assert.equal(first.nextCursor, 1);
+ db.db.exec("DROP TABLE analytics_events");
+
+ assert.throws(
+ () =>
+ db.backfillAnalyticsHistoryBatch({
+ throughId: state.targetId,
+ checkpointVersion: ANALYTICS_HISTORY_BACKFILL_VERSION,
+ limit: 1,
+ }),
+ /analytics_events/
+ );
+ assert.equal(db.getAnalyticsHistoryBackfillState().scannedThroughId, 1);
+});
+
+test("clearing history keeps old counters gone and admits later transcription IDs", (t) => {
+ const db = createDb(t);
+ if (!db) return;
+
+ db.saveTranscription("old checkpoint words", null, {
+ clientTranscriptionId: "checkpoint-before-clear",
+ analyticsOccurredAt: "2026-09-01T10:00:00.000Z",
+ });
+ reconcileThroughCheckpoint(db);
+ assert.equal(db.getAnalyticsSummary().totalDictations, 1);
+
+ db.clearTranscriptions();
+ assert.equal(db.getAnalyticsSummary().totalDictations, 0);
+ const afterClear = new Date(Date.now() + 60_000).toISOString();
+ db.saveTranscription("new checkpoint words", null, {
+ clientTranscriptionId: "checkpoint-after-clear",
+ analyticsOccurredAt: afterClear,
+ });
+ assert.equal(reconcileThroughCheckpoint(db)[0].inserted, 1);
+ assert.equal(db.getAnalyticsSummary().totalDictations, 1);
+ assert.equal(
+ db.db
+ .prepare("SELECT COUNT(*) AS count FROM analytics_events WHERE event_id = ?")
+ .get("checkpoint-before-clear").count,
+ 0
+ );
+});
+
+test("a dictation whose time cannot be read stays out instead of landing on today", (t) => {
+ const db = createDb(t);
+ if (!db) return;
+
+ db.db
+ .prepare(
+ `INSERT INTO transcriptions (
+ text, status, client_transcription_id, timestamp, created_at
+ ) VALUES ('undateable words', 'completed', 'no-usable-time', 'not-a-time', 'not-a-time')`
+ )
+ .run();
+
+ assert.deepEqual(db.backfillAnalyticsHistoryBatch(), {
+ complete: true,
+ nextCursor: 1,
+ scanned: 1,
+ inserted: 0,
+ skipped: 1,
+ });
+ const summary = db.getAnalyticsSummary();
+ assert.equal(summary.totalDictations, 0, "an undateable row must not become today's dictation");
+ assert.equal(summary.currentStreakDays, 0, "and must not manufacture a streak");
+});
+
+// SQLite reads a bare YYYY-MM-DD as carrying a zone, because the day hyphen
+// sits six characters from the end, so the query's shape test admits a row the
+// JS side then dates from created_at instead. Nothing writes that shape today,
+// but the boundary has to hold on the instant actually written, not on the
+// column the query happened to filter.
+test("history the user cleared cannot come back through a mis-shaped timestamp", (t) => {
+ const db = createDb(t);
+ if (!db) return;
+
+ db.db
+ .prepare(
+ `INSERT INTO transcriptions (
+ text, status, client_transcription_id, timestamp, created_at
+ ) VALUES ('cleared words', 'completed', 'shape-bypass', '2027-01-01',
+ '2026-01-02 03:04:05')`
+ )
+ .run();
+ db.db
+ .prepare(
+ "INSERT INTO analytics_device_clear_state (id, cleared_through) VALUES (1, '2026-08-15T00:00:00.000Z')"
+ )
+ .run();
+
+ assert.deepEqual(db.backfillAnalyticsHistoryBatch(), {
+ complete: true,
+ nextCursor: 1,
+ scanned: 1,
+ inserted: 0,
+ skipped: 1,
+ });
+ assert.equal(db.getAnalyticsSummary().totalDictations, 0);
+});
+
+// upsertTranscriptionFromCloud carries the cloud created_at but lets timestamp
+// default to the local pull, so trusting a naive timestamp would date every
+// pulled dictation to the day this device happened to sync.
+test("a cloud-pulled dictation is dated from its creation time, not the pull", (t) => {
+ const db = createDb(t);
+ if (!db) return;
+
+ db.upsertTranscriptionFromCloud({
+ client_transcription_id: "pulled-history",
+ id: "cloud-1",
+ text: "three pulled words",
+ status: "completed",
+ created_at: "2026-03-04 08:00:00",
+ });
+
+ assert.equal(db.backfillAnalyticsHistoryBatch().inserted, 1);
+ assert.equal(
+ db.db.prepare("SELECT local_date FROM analytics_events WHERE event_id = 'pulled-history'").get()
+ .local_date,
+ localDateKey(new Date("2026-03-04T08:00:00Z"))
+ );
+});
+
+test("backfill preserves the transcription creation time for retention", (t) => {
+ const db = createDb(t);
+ if (!db) return;
+
+ db.db
+ .prepare(
+ `INSERT INTO transcriptions (
+ text, status, client_transcription_id, timestamp, created_at
+ ) VALUES ('old words', 'completed', 'old-retained',
+ '2020-01-01T09:59:00.000Z', '2020-01-01 10:00:00')`
+ )
+ .run();
+
+ assert.equal(db.backfillAnalyticsHistoryBatch().inserted, 1);
+ assert.equal(
+ db.db.prepare("SELECT created_at FROM analytics_events WHERE event_id = 'old-retained'").get()
+ .created_at,
+ "2020-01-01 10:00:00"
+ );
+ assert.equal(db.deleteTranscriptionsExpiredBefore(30).analyticsPurged, 1);
+ assert.equal(db.getAnalyticsSummary().totalDictations, 0);
+});
+
+test("new transcription rows retain the original analytics occurrence time", (t) => {
+ const db = createDb(t);
+ if (!db) return;
+
+ const occurredAt = "2026-09-01T09:58:00.000Z";
+ const result = db.saveTranscription("saved words", "saved words", {
+ analyticsOccurredAt: occurredAt,
+ });
+ assert.equal(result.transcription.timestamp, occurredAt.replace("T", " "));
+});
+
test("analytics stays content-free and idempotent, and only syncs the signed-in account", (t) => {
const db = createDb(t);
if (!db) return;
@@ -260,6 +841,92 @@ test("the pending batch carries both the precise timestamp and the local date",
}
});
+test("pending analytics prioritize live events over historical rollback retries", (t) => {
+ const db = createDb(t);
+ if (!db) return;
+
+ db.setActiveAccountId("account-a");
+ recordEvent(db, "old-history", { occurredAt: "2026-01-01T09:00:00.000Z" });
+ recordEvent(db, "new-live", { occurredAt: "2026-08-30T09:00:00.000Z" });
+ db.db
+ .prepare("UPDATE analytics_events SET counter_version = 0 WHERE event_id = 'old-history'")
+ .run();
+
+ assert.deepEqual(
+ db.getPendingAnalyticsEvents(1).map((row) => row.event_id),
+ ["new-live"],
+ "an old API rejecting version-zero history must not hold live analytics behind it"
+ );
+});
+
+test("a dictation pulled from the cloud is dated when it was spoken, not when it synced", (t) => {
+ const db = createDb(t);
+ if (!db) return;
+
+ // The history list sorts on timestamp. Leaving cloud rows to default it makes
+ // a whole pull land at "now" and sort above dictations spoken since, so an
+ // old archive arrives on top of this morning's work.
+ db.upsertTranscriptionFromCloud({
+ client_transcription_id: "from-cloud",
+ id: "cloud-1",
+ text: "spoken last spring",
+ created_at: "2026-04-01T09:00:00.000Z",
+ });
+
+ const [row] = db.getTranscriptions(10);
+ assert.equal(row.client_transcription_id, "from-cloud");
+ assert.equal(row.timestamp, "2026-04-01 09:00:00.000Z");
+});
+
+test("a cloud pull sorts by when it was spoken, in the shape the API actually sends", (t) => {
+ const db = createDb(t);
+ if (!db) return;
+
+ // The API serializes transcriptions.created_at, a timestamptz, straight to
+ // JSON, so it arrives ISO 8601 with a "T". The history list sorts timestamp
+ // as TEXT and "T" (0x54) outranks the space (0x20) every locally written row
+ // uses, so an unnormalized cloud value jumps above everything spoken on the
+ // device that day regardless of the hour.
+ db.saveTranscription("spoken here tonight", null, {
+ clientTranscriptionId: "local-late",
+ analyticsOccurredAt: "2026-04-01T23:00:00.000Z",
+ });
+ db.upsertTranscriptionFromCloud({
+ client_transcription_id: "cloud-early",
+ id: "cloud-2",
+ text: "spoken elsewhere this morning",
+ created_at: "2026-04-01T02:00:00.000Z",
+ });
+
+ assert.deepEqual(
+ db.getTranscriptions(10).map((row) => row.client_transcription_id),
+ ["local-late", "cloud-early"],
+ "newest first, whichever writer stored the row"
+ );
+});
+
+test("a cloud pull does not overwrite the local recording time it already has", (t) => {
+ const db = createDb(t);
+ if (!db) return;
+
+ // The device that made the dictation recorded when speech started; the cloud
+ // only knows when the row was created. The local value is the better one.
+ db.saveTranscription("spoken here", null, {
+ clientTranscriptionId: "local-first",
+ analyticsOccurredAt: "2026-04-01T09:00:00.000Z",
+ });
+ db.upsertTranscriptionFromCloud({
+ client_transcription_id: "local-first",
+ id: "cloud-2",
+ text: "spoken here, cleaned",
+ created_at: "2026-04-02 17:30:00",
+ });
+
+ const [row] = db.getTranscriptions(10);
+ assert.equal(row.timestamp, "2026-04-01 09:00:00.000Z");
+ assert.equal(row.text, "spoken here, cleaned", "the pull still updates the transcript");
+});
+
test("the opt-in count covers everything turning sync on would upload", (t) => {
const db = createDb(t);
if (!db) return;
diff --git a/test/helpers/audioManagerOccurrenceTime.test.js b/test/helpers/audioManagerOccurrenceTime.test.js
new file mode 100644
index 000000000..d3b482ebd
--- /dev/null
+++ b/test/helpers/audioManagerOccurrenceTime.test.js
@@ -0,0 +1,54 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+const { loadAudioManager } = require("./harness/audioManager");
+
+// transcriptions.timestamp is what the history list sorts and groups on. A
+// successful dictation stores when speech started, so every other row a
+// recording can produce has to be dated the same way -- otherwise a failed
+// take sorts above the dictation that was still being transcribed when it
+// happened, and the column means two different things depending on the row.
+async function loadManagerClass(t) {
+ return loadAudioManager(t, {
+ cachePrefix: "openwhispr-occurrence-time-test-",
+ settingsKey: "__occurrenceTimeSettings",
+ settings: { dataRetentionEnabled: true, audioRetentionDays: 0 },
+ });
+}
+
+test("a failed take is dated when it was recorded, not when the failure was written", async (t) => {
+ const { AudioManager, window } = await loadManagerClass(t);
+ const saved = [];
+ window.electronAPI.saveTranscription = async (text, rawText, options) => {
+ saved.push(options);
+ return { id: 1 };
+ };
+
+ const manager = Object.assign(Object.create(AudioManager.prototype), {
+ lastAudioBlob: null,
+ lastAudioMetadata: null,
+ translationRequested: false,
+ });
+ await manager.saveFailedTranscription("boom", null, {
+ analyticsOccurredAt: "2026-04-01T09:00:00.000Z",
+ });
+
+ assert.equal(saved.length, 1);
+ assert.equal(saved[0].analyticsOccurredAt, "2026-04-01T09:00:00.000Z");
+});
+
+test("a discarded take carries its recording time past the state reset", async (t) => {
+ const { AudioManager } = await loadManagerClass(t);
+ const manager = Object.assign(Object.create(AudioManager.prototype), {
+ recordingStartTime: Date.parse("2026-04-01T09:00:00.000Z"),
+ audioChunks: [],
+ _batchSegments: [],
+ recordingMimeType: "audio/webm",
+ });
+
+ // The snapshot exists because the save is async and runs after the manager
+ // has already been reset for the next recording, so recordingStartTime is
+ // gone by then -- the occurrence time has to be captured with the audio.
+ const snapshot = manager.takeDiscardedBatchSnapshot();
+
+ assert.equal(snapshot.analyticsOccurredAt, "2026-04-01T09:00:00.000Z");
+});
diff --git a/test/helpers/audioStorageFilename.test.js b/test/helpers/audioStorageFilename.test.js
new file mode 100644
index 000000000..c0b197900
--- /dev/null
+++ b/test/helpers/audioStorageFilename.test.js
@@ -0,0 +1,34 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+
+const { installElectronStub } = require("./harness/electronStub.js");
+
+installElectronStub();
+const AudioStorageManager = require("../../src/helpers/audioStorage.js");
+
+// Saved audio is named after the dictation's wall clock, so the name has to be
+// read in the user's zone. transcriptions.timestamp comes in two shapes: the
+// bare SQLite CURRENT_TIMESTAMP form, which is UTC with no designator, and the
+// zoned form saveTranscription now writes. Only the second is unambiguous to
+// `new Date`, so the first has to be pinned to UTC before it is read.
+function filenameFor(timestamp) {
+ return Object.create(AudioStorageManager.prototype)._buildFilename(42, timestamp);
+}
+
+test("names saved audio in local time whether or not the timestamp carries a zone", (t) => {
+ const original = process.env.TZ;
+ process.env.TZ = "America/Los_Angeles";
+ t.after(() => {
+ process.env.TZ = original;
+ });
+
+ // 12:00 UTC is 05:00 in Los Angeles. Both spellings name the same instant, so
+ // both must produce the same name.
+ assert.equal(filenameFor("2026-09-10 12:00:00.000Z"), "OpenWhispr-2026-09-10-05-00-00-42.webm");
+ assert.equal(filenameFor("2026-09-10 12:00:00"), "OpenWhispr-2026-09-10-05-00-00-42.webm");
+});
+
+test("falls back to the bare name when the timestamp is unusable", () => {
+ assert.equal(filenameFor(null), "OpenWhispr-42.webm");
+ assert.equal(filenameFor("not a date"), "OpenWhispr-42.webm");
+});
diff --git a/test/helpers/policyRules.test.js b/test/helpers/policyRules.test.js
index 23c010dc6..97d8d0da0 100644
--- a/test/helpers/policyRules.test.js
+++ b/test/helpers/policyRules.test.js
@@ -1014,3 +1014,19 @@ test("the enterprise transcription tile is offered only to a managed policy snap
["openwhispr", "providers", "local", "self-hosted", "enterprise"]
);
});
+
+test("the local-history policy is resolved only once the fetch has settled", async () => {
+ const { isLocalHistoryPolicyResolved } = await load();
+ const snapshot = (status) => ({ status, policy: null, appVersion: null });
+
+ // Settled: the org either locks the switch or it does not.
+ assert.equal(isLocalHistoryPolicyResolved(snapshot("managed")), true);
+ assert.equal(isLocalHistoryPolicyResolved(snapshot("unmanaged")), true);
+
+ // Unsettled. effectiveLocalHistoryEnabled resolves these to the user's own
+ // preference, which is the right value to show and to sweep retention with,
+ // but it is a default rather than an answer -- so it is not consent.
+ assert.equal(isLocalHistoryPolicyResolved(snapshot("idle")), false);
+ assert.equal(isLocalHistoryPolicyResolved(snapshot("loading")), false);
+ assert.equal(isLocalHistoryPolicyResolved(snapshot("error")), false);
+});
diff --git a/test/helpers/retentionSettings.test.js b/test/helpers/retentionSettings.test.js
index 6b8ab32be..5b7b63c04 100644
--- a/test/helpers/retentionSettings.test.js
+++ b/test/helpers/retentionSettings.test.js
@@ -15,7 +15,12 @@ test("reports a change when a retention period is shortened", () => {
}),
{
changed: true,
- settings: { audioRetentionDays: 1, transcriptRetentionDays: 1 },
+ settings: {
+ audioRetentionDays: 1,
+ transcriptRetentionDays: 1,
+ dataRetentionEnabled: true,
+ localHistoryPolicyResolved: false,
+ },
}
);
});
@@ -29,7 +34,12 @@ test("is idempotent when both values are unchanged — dual-window mount sync",
});
test("keeps the current value when an incoming value is missing or unusable", () => {
- const current = { audioRetentionDays: 7, transcriptRetentionDays: 1 };
+ const current = {
+ audioRetentionDays: 7,
+ transcriptRetentionDays: 1,
+ dataRetentionEnabled: true,
+ localHistoryPolicyResolved: false,
+ };
for (const incoming of [
undefined,
{},
@@ -45,7 +55,12 @@ test("keeps the current value when an incoming value is missing or unusable", ()
test("only the main renderer can replace process-global retention settings", () => {
const mainRenderer = {};
const auxiliaryRenderers = [{}, {}, {}];
- const managedSettings = { audioRetentionDays: 7, transcriptRetentionDays: 30 };
+ const managedSettings = {
+ audioRetentionDays: 7,
+ transcriptRetentionDays: 30,
+ dataRetentionEnabled: true,
+ localHistoryPolicyResolved: false,
+ };
let current = { ...DEFAULT_RETENTION_SETTINGS };
let cleanupRuns = 0;
let synced = false;
@@ -77,7 +92,12 @@ test("only the main renderer can replace process-global retention settings", ()
{ sender: mainRenderer },
{ audioRetentionDays: 90, transcriptRetentionDays: 0 }
);
- assert.deepEqual(current, { audioRetentionDays: 90, transcriptRetentionDays: 0 });
+ assert.deepEqual(current, {
+ audioRetentionDays: 90,
+ transcriptRetentionDays: 0,
+ dataRetentionEnabled: true,
+ localHistoryPolicyResolved: false,
+ });
assert.equal(cleanupRuns, 2);
});
@@ -141,5 +161,53 @@ test("a disabled retention setting reaches the sweep before it can delete", () =
});
handle({ sender: owner }, { audioRetentionDays: 0, transcriptRetentionDays: 0 });
- assert.deepEqual(sweptWith, [{ audioRetentionDays: 0, transcriptRetentionDays: 0 }]);
+ assert.deepEqual(sweptWith, [
+ {
+ audioRetentionDays: 0,
+ transcriptRetentionDays: 0,
+ dataRetentionEnabled: true,
+ localHistoryPolicyResolved: false,
+ },
+ ]);
+});
+
+test("carries the effective local-history switch, and reports it changing", () => {
+ const current = { ...DEFAULT_RETENTION_SETTINGS, dataRetentionEnabled: true };
+
+ const off = applyRetentionSettings(current, {
+ audioRetentionDays: current.audioRetentionDays,
+ transcriptRetentionDays: current.transcriptRetentionDays,
+ dataRetentionEnabled: false,
+ });
+ assert.equal(off.settings.dataRetentionEnabled, false);
+ assert.equal(off.changed, true, "turning local history off is a change worth acting on");
+
+ // A renderer that predates this field must not be read as "history off".
+ const legacy = applyRetentionSettings(current, {
+ audioRetentionDays: current.audioRetentionDays,
+ transcriptRetentionDays: current.transcriptRetentionDays,
+ });
+ assert.equal(legacy.settings.dataRetentionEnabled, true);
+ assert.equal(legacy.changed, false);
+});
+
+test("carries whether the local-history policy has resolved, and reports it changing", () => {
+ assert.equal(
+ DEFAULT_RETENTION_SETTINGS.localHistoryPolicyResolved,
+ false,
+ "an unreported policy must never read as a resolved one"
+ );
+
+ const arrival = applyRetentionSettings(DEFAULT_RETENTION_SETTINGS, {
+ audioRetentionDays: 30,
+ transcriptRetentionDays: 0,
+ dataRetentionEnabled: true,
+ localHistoryPolicyResolved: true,
+ });
+ assert.equal(
+ arrival.changed,
+ true,
+ "the policy settling is the change that unblocks reconstruction"
+ );
+ assert.equal(arrival.settings.localHistoryPolicyResolved, true);
});
diff --git a/test/helpers/syncScenarios.test.js b/test/helpers/syncScenarios.test.js
index a74c24f2e..d3a8397c5 100644
--- a/test/helpers/syncScenarios.test.js
+++ b/test/helpers/syncScenarios.test.js
@@ -326,9 +326,8 @@ test("S2 two devices: an older empty shell never destroys the transcript and the
const mark2 = cloud.log.length;
await syncOnce(deviceA);
assert.equal(
- cloud.log
- .slice(mark2)
- .filter((c) => c.method === "PATCH" && c.path === "/api/notes/update").length,
+ cloud.log.slice(mark2).filter((c) => c.method === "PATCH" && c.path === "/api/notes/update")
+ .length,
0,
"a conflicted row must not be re-pushed before the user resolves it"
);
diff --git a/test/services/analyticsService.test.js b/test/services/analyticsService.test.js
index de60e8659..5df062973 100644
--- a/test/services/analyticsService.test.js
+++ b/test/services/analyticsService.test.js
@@ -300,6 +300,104 @@ test("account analytics sends UTC when the runtime exposes no timezone", async (
assert.equal(requestUrl.searchParams.get("timeZone"), "UTC");
});
+test("account analytics requests history once per account and not on ordinary refreshes", async (t) => {
+ const requests = [];
+ installBrowserGlobals(t, {
+ window: {
+ electronAPI: {
+ cloudApiRequest: async (request) => {
+ requests.push(request);
+ return {
+ success: true,
+ data: { ...VALID_SUMMARY, scope: "account", timeZone: "UTC" },
+ };
+ },
+ },
+ },
+ });
+ const vite = await createRendererServer(t);
+ const { getAccountAnalyticsSummary } = await vite.ssrLoadModule("/services/AnalyticsService.ts");
+
+ await getAccountAnalyticsSummary("account-a");
+ await getAccountAnalyticsSummary("account-a");
+ await getAccountAnalyticsSummary("account-b");
+ await getAccountAnalyticsSummary();
+
+ assert.deepEqual(
+ requests.map((request) =>
+ new URL(request.path, "https://api.openwhispr.com").searchParams.get("backfill")
+ ),
+ ["true", null, "true", null]
+ );
+});
+
+test("account analytics retries the history trigger after a failed request", async (t) => {
+ const requests = [];
+ installBrowserGlobals(t, {
+ window: {
+ electronAPI: {
+ cloudApiRequest: async (request) => {
+ requests.push(request);
+ if (requests.length === 1) {
+ return { success: false, status: 503, error: "temporarily unavailable" };
+ }
+ return {
+ success: true,
+ data: { ...VALID_SUMMARY, scope: "account", timeZone: "UTC" },
+ };
+ },
+ },
+ },
+ });
+ const vite = await createRendererServer(t);
+ const { getAccountAnalyticsSummary } = await vite.ssrLoadModule("/services/AnalyticsService.ts");
+
+ await assert.rejects(getAccountAnalyticsSummary("retry-account"));
+ await getAccountAnalyticsSummary("retry-account");
+
+ assert.deepEqual(
+ requests.map((request) =>
+ new URL(request.path, "https://api.openwhispr.com").searchParams.get("backfill")
+ ),
+ ["true", "true"]
+ );
+});
+
+test("account analytics retries the history trigger when queue scheduling is unavailable", async (t) => {
+ const requests = [];
+ installBrowserGlobals(t, {
+ window: {
+ electronAPI: {
+ cloudApiRequest: async (request) => {
+ requests.push(request);
+ return {
+ success: true,
+ data: {
+ ...VALID_SUMMARY,
+ historyBackfillRetryRequired: requests.length === 1,
+ },
+ };
+ },
+ },
+ },
+ });
+ const vite = await createRendererServer(t);
+ const { getAccountAnalyticsSummary } = await vite.ssrLoadModule("/services/AnalyticsService.ts");
+
+ assert.equal(
+ (await getAccountAnalyticsSummary("queue-retry-account")).historyBackfillRetryRequired,
+ true
+ );
+ await getAccountAnalyticsSummary("queue-retry-account");
+
+ assert.deepEqual(
+ requests.map((request) =>
+ new URL(request.path, "https://api.openwhispr.com").searchParams.get("backfill")
+ ),
+ ["true", "true"]
+ );
+});
+
for (const [name, daily] of [
["a null bucket", [null]],
["an impossible date", [{ ...VALID_SUMMARY.daily[0], date: "2026-02-30" }]],
@@ -327,6 +425,37 @@ for (const [name, daily] of [
});
}
+test("a malformed successful summary does not retrigger server history", async (t) => {
+ const requests = [];
+ installBrowserGlobals(t, {
+ window: {
+ electronAPI: {
+ cloudApiRequest: async (request) => {
+ requests.push(request);
+ return requests.length === 1
+ ? { success: true, data: { ...VALID_SUMMARY, daily: [null] } }
+ : { success: true, data: VALID_SUMMARY };
+ },
+ },
+ },
+ });
+ const vite = await createRendererServer(t);
+ const { getAccountAnalyticsSummary } = await vite.ssrLoadModule("/services/AnalyticsService.ts");
+
+ await assert.rejects(
+ getAccountAnalyticsSummary("malformed-account"),
+ /Malformed analytics summary from cloud/
+ );
+ await getAccountAnalyticsSummary("malformed-account");
+
+ assert.deepEqual(
+ requests.map((request) =>
+ new URL(request.path, "https://api.openwhispr.com").searchParams.get("backfill")
+ ),
+ ["true", null]
+ );
+});
+
test("analytics refreshes locally on change and remotely only while cloud Insights are active", async (t) => {
const windowListeners = new Map();
const documentListeners = new Map();
@@ -708,6 +837,53 @@ test("revoking upload consent stops a multi-batch drain after its active request
assert.equal(retired.has("event-200"), false, "the next batch stays pending after revocation");
});
+test("a history backlog is capped per pass and drains across later passes", async (t) => {
+ // Insights waits on a pass before it can read the account summary, so an
+ // unbounded drain held the view's spinner — and hammered the batch endpoint —
+ // for as long as a whole reconstructed history took to upload.
+ const pending = Array.from({ length: 1400 }, (_, index) => ({
+ ...EVENT,
+ event_id: `event-${index}`,
+ counter_version: 0,
+ }));
+ const retired = new Set();
+ const posted = [];
+ installBrowserGlobals(t, {
+ window: {
+ electronAPI: {
+ getPendingAnalyticsClear: async () => null,
+ getPendingAnalyticsDeletes: async () => [],
+ getPendingAnalyticsEvents: async (limit) =>
+ pending.filter((event) => !retired.has(event.event_id)).slice(0, limit),
+ markAnalyticsEventsSynced: async (eventIds) => {
+ eventIds.forEach((eventId) => retired.add(eventId));
+ return { success: true, updated: eventIds.length };
+ },
+ cloudApiRequest: async (request) => {
+ posted.push(request.body.events.length);
+ return {
+ success: true,
+ data: {
+ accepted: request.body.events.map((event) => event.event_id),
+ supportsHistoricalCounterVersion: true,
+ },
+ };
+ },
+ },
+ },
+ });
+ const vite = await createRendererServer(t);
+ const { syncPendingAnalytics } = await vite.ssrLoadModule("/services/AnalyticsService.ts");
+
+ assert.equal(await syncPendingAnalytics(), 1000, "a pass stops at its batch budget");
+ assert.equal(posted.length, 5);
+ assert.equal(retired.size, 1000, "the rest of the backlog stays pending");
+
+ assert.equal(await syncPendingAnalytics(), 400, "a later pass drains the remainder");
+ assert.equal(posted.length, 7);
+ assert.equal(retired.size, 1400);
+});
+
test("analytics queue and cloud operations carry one pinned account context", async (t) => {
const context = { accountId: "account-1", authGeneration: 17 };
const localCalls = [];
@@ -791,3 +967,186 @@ test("a withheld row is offered once per pass, not once per batch behind it", as
assert.deepEqual(posted, [["event-0", "event-1", "event-2"]]);
assert.equal(retired.has("event-0"), false, "the withheld row stays pending for a later pass");
});
+
+test("rejected version-zero history survives an API rollback and syncs after recovery", async (t) => {
+ const pending = [
+ { ...EVENT, event_id: "history-rejected-by-old-api", counter_version: 0 },
+ { ...EVENT, event_id: "history-stored", counter_version: 0 },
+ { ...EVENT, event_id: "invalid-live-event", counter_version: 1 },
+ ];
+ const retired = new Set();
+ let requests = 0;
+ installBrowserGlobals(t, {
+ window: {
+ electronAPI: {
+ getPendingAnalyticsClear: async () => null,
+ getPendingAnalyticsDeletes: async () => [],
+ getPendingAnalyticsEvents: async () =>
+ pending.filter((event) => !retired.has(event.event_id)),
+ markAnalyticsEventsSynced: async (eventIds) => {
+ eventIds.forEach((eventId) => retired.add(eventId));
+ return { success: true, updated: eventIds.length };
+ },
+ cloudApiRequest: async (request) => {
+ requests += 1;
+ const accepted = request.body.events.map((event) => event.event_id);
+ return {
+ success: true,
+ data:
+ requests === 1
+ ? {
+ accepted,
+ rejected: ["history-rejected-by-old-api", "invalid-live-event"],
+ }
+ : {
+ accepted,
+ rejected: [],
+ supportsHistoricalCounterVersion: true,
+ },
+ };
+ },
+ },
+ },
+ });
+ const vite = await createRendererServer(t);
+ const { syncPendingAnalytics } = await vite.ssrLoadModule("/services/AnalyticsService.ts");
+
+ const realNow = Date.now;
+ t.after(() => {
+ Date.now = realNow;
+ });
+
+ assert.equal(await syncPendingAnalytics(), 2);
+ assert.deepEqual([...retired].sort(), ["history-stored", "invalid-live-event"]);
+
+ // Refused history backs off instead of riding every pass, so recovery lands
+ // on the first pass after that window rather than on the very next one.
+ Date.now = () => realNow() + 2 * 60 * 60 * 1000;
+
+ assert.equal(await syncPendingAnalytics(), 1);
+ assert.equal(retired.has("history-rejected-by-old-api"), true);
+});
+
+test("an API without version-zero support is not re-asked on every pass", async (t) => {
+ const pending = [{ ...EVENT, event_id: "history-row", counter_version: 0 }];
+ const posts = [];
+ installBrowserGlobals(t, {
+ window: {
+ electronAPI: {
+ getPendingAnalyticsClear: async () => null,
+ getPendingAnalyticsDeletes: async () => [],
+ getPendingAnalyticsEvents: async () => pending,
+ markAnalyticsEventsSynced: async (eventIds) => ({
+ success: true,
+ updated: eventIds.length,
+ }),
+ cloudApiRequest: async (request) => {
+ posts.push(request.body.events.map((event) => event.event_id));
+ // The shipped API before version-zero support: the row fails
+ // validation, so it is echoed into both lists and never stored.
+ return {
+ success: true,
+ data: { accepted: ["history-row"], rejected: ["history-row"] },
+ };
+ },
+ },
+ },
+ });
+ const vite = await createRendererServer(t);
+ const { syncPendingAnalytics } = await vite.ssrLoadModule("/services/AnalyticsService.ts");
+
+ await syncPendingAnalytics();
+ await syncPendingAnalytics();
+ await syncPendingAnalytics();
+
+ assert.deepEqual(
+ posts,
+ [["history-row"]],
+ "one refusal is enough -- the row stays pending, but stops riding every pass"
+ );
+});
+
+// getPendingAnalyticsEvents orders (counter_version = 0) ASC, occurred_at ASC,
+// so live events always precede history. The early return in runAnalyticsPass
+// depends on that, so the stub has to reproduce it rather than insertion order.
+const inQueueOrder = (events) =>
+ [...events].sort(
+ (a, b) =>
+ Number(a.counter_version === 0) - Number(b.counter_version === 0) ||
+ a.occurred_at.localeCompare(b.occurred_at)
+ );
+
+test("live events keep uploading while history is backing off", async (t) => {
+ const retired = new Set();
+ const pending = [
+ { ...EVENT, event_id: "live-row", counter_version: 1 },
+ { ...EVENT, event_id: "history-row", counter_version: 0 },
+ ];
+ const posts = [];
+ installBrowserGlobals(t, {
+ window: {
+ electronAPI: {
+ getPendingAnalyticsClear: async () => null,
+ getPendingAnalyticsDeletes: async () => [],
+ getPendingAnalyticsEvents: async () =>
+ inQueueOrder(pending.filter((event) => !retired.has(event.event_id))),
+ markAnalyticsEventsSynced: async (eventIds) => {
+ eventIds.forEach((eventId) => retired.add(eventId));
+ return { success: true, updated: eventIds.length };
+ },
+ cloudApiRequest: async (request) => {
+ const ids = request.body.events.map((event) => event.event_id);
+ posts.push(ids);
+ return {
+ success: true,
+ data: { accepted: ids, rejected: ids.filter((id) => id === "history-row") },
+ };
+ },
+ },
+ },
+ });
+ const vite = await createRendererServer(t);
+ const { syncPendingAnalytics } = await vite.ssrLoadModule("/services/AnalyticsService.ts");
+
+ await syncPendingAnalytics();
+ retired.delete("live-row");
+ pending.push({ ...EVENT, event_id: "live-row-2", counter_version: 1 });
+ await syncPendingAnalytics();
+
+ assert.deepEqual(
+ posts[1],
+ ["live-row", "live-row-2"],
+ "the second pass carries current activity and leaves the refused history behind"
+ );
+});
+
+test("a capable API can permanently retire invalid version-zero history", async (t) => {
+ const event = { ...EVENT, event_id: "permanently-invalid-history", counter_version: 0 };
+ const retired = new Set();
+ installBrowserGlobals(t, {
+ window: {
+ electronAPI: {
+ getPendingAnalyticsClear: async () => null,
+ getPendingAnalyticsDeletes: async () => [],
+ getPendingAnalyticsEvents: async () => (retired.has(event.event_id) ? [] : [event]),
+ markAnalyticsEventsSynced: async (eventIds) => {
+ eventIds.forEach((eventId) => retired.add(eventId));
+ return { success: true, updated: eventIds.length };
+ },
+ cloudApiRequest: async () => ({
+ success: true,
+ data: {
+ accepted: [event.event_id],
+ rejected: [event.event_id],
+ supportsHistoricalCounterVersion: true,
+ },
+ }),
+ },
+ },
+ });
+ const vite = await createRendererServer(t);
+ const { syncPendingAnalytics } = await vite.ssrLoadModule("/services/AnalyticsService.ts");
+
+ assert.equal(await syncPendingAnalytics(), 1);
+ assert.equal(retired.has(event.event_id), true);
+});
diff --git a/test/services/leaderboardService.test.js b/test/services/leaderboardService.test.js
index 8a2c76a0a..0b8eafde7 100644
--- a/test/services/leaderboardService.test.js
+++ b/test/services/leaderboardService.test.js
@@ -61,7 +61,10 @@ test("participation uses the account endpoint for reads, joins, and leaves", asy
{
method: "PATCH",
path: "/api/analytics/participation",
- body: { enabled: true },
+ body: {
+ enabled: true,
+ timeZone: Intl.DateTimeFormat().resolvedOptions().timeZone || "UTC",
+ },
public: false,
expectedAuthGeneration: 7,
},
@@ -547,7 +550,13 @@ test("an explicit join stays newer than a pending leave already in flight", asyn
requests.map(({ method, body }) => ({ method, body })),
[
{ method: "PATCH", body: { enabled: false } },
- { method: "PATCH", body: { enabled: true } },
+ {
+ method: "PATCH",
+ body: {
+ enabled: true,
+ timeZone: Intl.DateTimeFormat().resolvedOptions().timeZone || "UTC",
+ },
+ },
{ method: "GET", body: undefined },
]
);
diff --git a/test/stores/leaderboardParticipationStore.test.js b/test/stores/leaderboardParticipationStore.test.js
index 82b1caea2..fff5102f8 100644
--- a/test/stores/leaderboardParticipationStore.test.js
+++ b/test/stores/leaderboardParticipationStore.test.js
@@ -223,7 +223,13 @@ test("a join retires the queued leave before its own request goes out", async (t
{
method: "PATCH",
path: "/api/analytics/participation",
- body: { enabled: true },
+ // The zone rides along with the join because that is what starts the
+ // account's history reconciliation, and only this device knows which
+ // calendar day its dictations belong to.
+ body: {
+ enabled: true,
+ timeZone: Intl.DateTimeFormat().resolvedOptions().timeZone || "UTC",
+ },
public: false,
expectedAuthGeneration: 7,
},
@@ -238,6 +244,18 @@ test("a join retires the queued leave before its own request goes out", async (t
assert.equal(store.getState().error, null);
});
+test("a join sends UTC when the runtime exposes no timezone", async (t) => {
+ const { context, requests, store } = await loadStore(t, {
+ cloudApiRequest: async () => participation(true),
+ });
+ t.mock.method(Intl, "DateTimeFormat", () => ({
+ resolvedOptions: () => ({ timeZone: "" }),
+ }));
+
+ assert.equal(await store.getState().join(context), true);
+ assert.deepEqual(requests[0].body, { enabled: true, timeZone: "UTC" });
+});
+
test("a failed join reports that participation did not change", async (t) => {
const { context, storage, store } = await loadStore(t, {
cloudApiRequest: async () => ({ success: false, status: 500, error: "server error" }),