Skip to content

Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.

perf(server): share concurrent usage reads and pricing refreshes (#200)

Leo8c9c426bbb8b
diff --git a/apps/server/src/usage/UsageService.test.ts b/apps/server/src/usage/UsageService.test.ts
new file mode 100644
index 000000000..37ef63df4
--- /dev/null
+++ b/apps/server/src/usage/UsageService.test.ts
@@ -0,0 +1,373 @@
+import * as NodeServices from "@effect/platform-node/NodeServices";
+import { expect, it } from "@effect/vitest";
+import { UsageDay, type UsageSummaryInput } from "@t3tools/contracts";
+import * as Schema from "effect/Schema";
+import * as Clock from "effect/Clock";
+import * as Deferred from "effect/Deferred";
+import * as Effect from "effect/Effect";
+import * as Exit from "effect/Exit";
+import * as Fiber from "effect/Fiber";
+import * as FileSystem from "effect/FileSystem";
+import * as Path from "effect/Path";
+import * as Scope from "effect/Scope";
+import * as TestClock from "effect/testing/TestClock";
+import { HttpClient, HttpClientResponse } from "effect/unstable/http";
+import { afterEach, vi } from "vite-plus/test";
+
+import * as ServerConfig from "../config.ts";
+import * as ServerSettings from "../serverSettings.ts";
+import * as UsageService from "./UsageService.ts";
+import * as TranscriptReader from "./usageTranscriptReader.ts";
+
+vi.mock("../subscription-auth/service.ts", () => ({
+ SubscriptionAuthService: {
+ forSecretsDir: () => ({ reload: () => {}, getPlanAccessToken: async () => undefined }),
+ },
+}));
+vi.mock("./usagePlanLimits.ts", () => ({ readPlanLimits: async () => [] }));
+
+afterEach(() => vi.restoreAllMocks());
+
+const DAY = 24 * 60 * 60 * 1000;
+const rateDocument = {
+ "claude-fable-5": { input_cost_per_token: 0.001, output_cost_per_token: 0.002 },
+};
+const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown));
+const input: UsageSummaryInput = {
+ sinceDay: UsageDay.make("2026-09-07"),
+ untilDay: UsageDay.make("2026-09-07"),
+ timeZone: "UTC",
+};
+const step = {
+ model: "claude-fable-5",
+ totals: {
+ uncachedInputTokens: 100,
+ cachedInputTokens: 0,
+ cacheCreationTokens: 0,
+ outputTokens: 20,
+ reasoningTokens: 0,
+ },
+ reportedCostUsd: null,
+};
+const transcript =
+ encodeJson({
+ type: "assistant",
+ timestamp: "2026-09-07T04:05:13.944Z",
+ sessionId: "session",
+ message: {
+ id: "message",
+ model: "claude-fable-5",
+ usage: { input_tokens: 100, output_tokens: 20 },
+ },
+ }) + "\n";
+
+const makeFixture = Effect.fnUntraced(function* (
+ options: {
+ response?: Effect.Effect<Response>;
+ diskRates?: { fetchedAtMs: number; document: unknown };
+ } = {},
+) {
+ const fs = yield* FileSystem.FileSystem;
+ const path = yield* Path.Path;
+ const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "akeru-usage-test-" });
+ const config = yield* ServerConfig.ServerConfig.pipe(
+ Effect.provide(ServerConfig.layerTest("/tmp", baseDir)),
+ );
+ const claude = path.join(config.baseDir, "claude");
+ const projects = path.join(claude, "projects");
+ yield* fs.makeDirectory(projects, { recursive: true });
+ const transcriptPath = path.join(projects, "session.jsonl");
+ yield* fs.writeFileString(transcriptPath, transcript);
+ const ratesPath = path.join(config.stateDir, "usage-model-rates.json");
+ if (options.diskRates) yield* fs.writeFileString(ratesPath, encodeJson(options.diskRates));
+ const persisted = yield* Deferred.make<void>();
+ let fetches = 0;
+ const client = HttpClient.make((request) =>
+ Effect.gen(function* () {
+ fetches += 1;
+ const response = yield* options.response ?? Effect.succeed(Response.json(rateDocument));
+ return HttpClientResponse.fromWeb(request, response);
+ }),
+ );
+ const service = yield* UsageService.make.pipe(
+ Effect.provideService(ServerConfig.ServerConfig, config),
+ Effect.provideService(HttpClient.HttpClient, client),
+ Effect.provideService(FileSystem.FileSystem, {
+ ...fs,
+ writeFileString: (target, contents, settings) =>
+ fs
+ .writeFileString(target, contents, settings)
+ .pipe(
+ Effect.tap(() =>
+ target === ratesPath ? Deferred.succeed(persisted, undefined) : Effect.void,
+ ),
+ ),
+ }),
+ Effect.provide(
+ ServerSettings.layerTest({
+ providers: {
+ claudeAgent: { homePath: claude },
+ codex: { homePath: path.join(config.baseDir, "codex") },
+ },
+ }),
+ ),
+ );
+ return {
+ service,
+ fetches: () => fetches,
+ persisted: Deferred.await(persisted),
+ transcriptPath,
+ fs,
+ };
+});
+
+it.layer(NodeServices.layer)("UsageService single-flight", (it) => {
+ it.effect("shares simultaneous cold summaries, transcript scans, and step pricing", () =>
+ Effect.gen(function* () {
+ const started = yield* Deferred.make<void>();
+ const reply = yield* Deferred.make<Response>();
+ const { service, fetches } = yield* makeFixture({
+ response: Deferred.succeed(started, undefined).pipe(Effect.andThen(Deferred.await(reply))),
+ });
+ const walks = vi.spyOn(TranscriptReader, "listTranscriptFiles");
+ const reads = vi.spyOn(TranscriptReader, "readTranscriptRecords");
+ const summaries = yield* Effect.all(
+ Array.from({ length: 8 }, () => service.readSummary(input)),
+ { concurrency: "unbounded" },
+ ).pipe(Effect.forkChild);
+ const prices = yield* Effect.all(
+ Array.from({ length: 8 }, () => service.priceStepUsage(step)),
+ { concurrency: "unbounded" },
+ ).pipe(Effect.forkChild);
+ yield* Deferred.await(started);
+ expect(fetches()).toBe(1);
+ yield* Deferred.succeed(reply, Response.json(rateDocument));
+ const results = yield* Fiber.join(summaries);
+ expect((yield* Fiber.join(prices)).every((price) => price.costSource === "modelPriced")).toBe(
+ true,
+ );
+ expect(results.every((result) => result === results[0])).toBe(true);
+ expect(walks).toHaveBeenCalledTimes(1);
+ expect(reads).toHaveBeenCalledTimes(1);
+ expect(results[0]!.pricing.status).toBe("fresh");
+ expect(fetches()).toBe(1);
+ yield* service.readSummary(input);
+ expect(walks).toHaveBeenCalledTimes(2);
+ expect(reads).toHaveBeenCalledTimes(1);
+ }),
+ );
+
+ it.effect("keeps a shared scan alive when its first caller disconnects", () =>
+ Effect.gen(function* () {
+ const started = yield* Deferred.make<void>();
+ const reply = yield* Deferred.make<Response>();
+ const { service, fetches } = yield* makeFixture({
+ response: Deferred.succeed(started, undefined).pipe(Effect.andThen(Deferred.await(reply))),
+ });
+ const first = yield* service.readSummary(input).pipe(Effect.forkChild);
+ yield* Deferred.await(started);
+ const remaining = yield* service.readSummary(input).pipe(Effect.forkChild);
+ yield* Fiber.interrupt(first);
+ yield* Deferred.succeed(reply, Response.json(rateDocument));
+ const summary = yield* Fiber.join(remaining);
+ expect(summary.pricing.status).toBe("fresh");
+ expect(summary.sources[0]!.scannedFiles).toBe(1);
+ expect(fetches()).toBe(1);
+ }),
+ );
+
+ it.effect("keeps a shared pricing refresh alive when its first step caller disconnects", () =>
+ Effect.gen(function* () {
+ const started = yield* Deferred.make<void>();
+ const reply = yield* Deferred.make<Response>();
+ const { service, fetches } = yield* makeFixture({
+ response: Deferred.succeed(started, undefined).pipe(Effect.andThen(Deferred.await(reply))),
+ });
+ const first = yield* service.priceStepUsage(step).pipe(Effect.forkChild);
+ yield* Deferred.await(started);
+ const remaining = yield* service.priceStepUsage(step).pipe(Effect.forkChild);
+ yield* Fiber.interrupt(first);
+ yield* Deferred.succeed(reply, Response.json(rateDocument));
+ expect((yield* Fiber.join(remaining)).costSource).toBe("modelPriced");
+ expect(fetches()).toBe(1);
+ }),
+ );
+
+ it.effect(
+ "shares per-file parsing across different summary windows without sharing aggregates",
+ () =>
+ Effect.gen(function* () {
+ const now = yield* Clock.currentTimeMillis;
+ const { service } = yield* makeFixture({
+ diskRates: { fetchedAtMs: now, document: rateDocument },
+ });
+ const reads = vi.spyOn(TranscriptReader, "readTranscriptRecords");
+ const results = yield* Effect.all(
+ [
+ service.readSummary(input),
+ service.readSummary({
+ ...input,
+ sinceDay: UsageDay.make("2026-09-06"),
+ timeZone: "America/New_York",
+ }),
+ ],
+ { concurrency: "unbounded" },
+ );
+ expect(reads).toHaveBeenCalledTimes(1);
+ expect(results[0]!.timeZone).toBe("UTC");
+ expect(results[1]!.timeZone).toBe("America/New_York");
+ }),
+ );
+
+ it.effect(
+ "shares failures, backs off repeated HTTP and malformed responses, and resets after success",
+ () =>
+ Effect.gen(function* () {
+ let response = () => new Response("offline", { status: 503 });
+ const { service, fetches } = yield* makeFixture({
+ response: Effect.sync(() => response()),
+ });
+ const failed = yield* Effect.all(
+ Array.from({ length: 8 }, () => service.readSummary(input)),
+ { concurrency: "unbounded" },
+ );
+ expect(fetches()).toBe(1);
+ expect(failed[0]!.pricing.status).toBe("unavailable");
+ yield* service.priceStepUsage(step);
+ yield* TestClock.adjust(59_999);
+ yield* service.priceStepUsage(step);
+ expect(fetches()).toBe(1);
+ yield* TestClock.adjust(1);
+ response = () => Response.json({ invalid: {} });
+ yield* service.priceStepUsage(step);
+ expect(fetches()).toBe(2);
+ yield* TestClock.adjust(119_999);
+ yield* service.priceStepUsage(step);
+ expect(fetches()).toBe(2);
+ yield* TestClock.adjust(1);
+ response = () => Response.json(rateDocument);
+ expect((yield* service.priceStepUsage(step)).costSource).toBe("modelPriced");
+ expect(fetches()).toBe(3);
+ expect((yield* service.readSummary(input)).pricing.status).toBe("fresh");
+ expect(fetches()).toBe(3);
+ }),
+ );
+
+ it.effect("shares a timed-out fetch and waits for the failure backoff before retrying", () =>
+ Effect.gen(function* () {
+ const started = yield* Deferred.make<void>();
+ let response: Effect.Effect<Response> = Effect.never;
+ const { service, fetches } = yield* makeFixture({
+ response: Deferred.succeed(started, undefined).pipe(
+ Effect.andThen(Effect.suspend(() => response)),
+ ),
+ });
+ const prices = yield* Effect.all(
+ Array.from({ length: 4 }, () => service.priceStepUsage(step)),
+ { concurrency: "unbounded" },
+ ).pipe(Effect.forkChild);
+ yield* Deferred.await(started);
+ yield* TestClock.adjust(10_000);
+ expect((yield* Fiber.join(prices)).every((price) => price.costSource === "unpriced")).toBe(
+ true,
+ );
+ expect((yield* service.readSummary(input)).pricing.status).toBe("unavailable");
+ yield* TestClock.adjust(59_999);
+ yield* service.priceStepUsage(step);
+ expect(fetches()).toBe(1);
+ response = Effect.succeed(Response.json(rateDocument));
+ yield* TestClock.adjust(1);
+ expect((yield* service.priceStepUsage(step)).costSource).toBe("modelPriced");
+ expect(fetches()).toBe(2);
+ }),
+ );
+
+ it.effect("interrupts a detached stale-pricing refresh when the service scope closes", () =>
+ Effect.gen(function* () {
+ yield* TestClock.setTime(2 * DAY);
+ const started = yield* Deferred.make<void>();
+ const stopped = yield* Deferred.make<void>();
+ const scope = yield* Scope.make();
+ const { service, fetches } = yield* makeFixture({
+ diskRates: { fetchedAtMs: 0, document: rateDocument },
+ response: Deferred.succeed(started, undefined).pipe(
+ Effect.andThen(Effect.never),
+ Effect.ensuring(Deferred.succeed(stopped, undefined)),
+ ),
+ }).pipe(Effect.provideService(Scope.Scope, scope));
+ expect((yield* service.priceStepUsage(step)).costSource).toBe("modelPriced");
+ yield* Deferred.await(started);
+ yield* Scope.close(scope, Exit.void);
+ yield* Deferred.await(stopped);
+ expect(fetches()).toBe(1);
+ }),
+ );
+
+ it.effect("serves stale prices immediately with cached status while one refresh is blocked", () =>
+ Effect.gen(function* () {
+ yield* TestClock.setTime(2 * DAY);
+ const started = yield* Deferred.make<void>();
+ const reply = yield* Deferred.make<Response>();
+ const { service, fetches, persisted } = yield* makeFixture({
+ diskRates: { fetchedAtMs: 0, document: rateDocument },
+ response: Deferred.succeed(started, undefined).pipe(Effect.andThen(Deferred.await(reply))),
+ });
+ const summary = yield* service.readSummary(input);
+ yield* Deferred.await(started);
+ expect(summary.pricing).toMatchObject({
+ status: "cached",
+ fetchedAt: "1970-01-01T00:00:00.000Z",
+ knownModels: 1,
+ });
+ const prices = yield* Effect.all(
+ Array.from({ length: 8 }, () => service.priceStepUsage(step)),
+ { concurrency: "unbounded" },
+ );
+ expect(prices.every((price) => price.costSource === "modelPriced")).toBe(true);
+ expect(fetches()).toBe(1);
+ yield* Deferred.succeed(reply, Response.json(rateDocument));
+ yield* persisted;
+ expect((yield* service.readSummary(input)).pricing.status).toBe("fresh");
+ }),
+ );
+
+ it.effect(
+ "marks a once-fresh table cached before failed refreshes and keeps its successful timestamp",
+ () =>
+ Effect.gen(function* () {
+ let response = () => Response.json(rateDocument);
+ const { service, fetches } = yield* makeFixture({
+ response: Effect.sync(() => response()),
+ });
+ const fresh = yield* service.readSummary(input);
+ yield* TestClock.adjust(DAY);
+ response = () => new Response("offline", { status: 503 });
+ const stale = yield* service.readSummary(input);
+ expect(stale.pricing.status).toBe("cached");
+ expect(stale.pricing.fetchedAt).toBe(fresh.pricing.fetchedAt);
+ expect(stale.pricing.knownModels).toBe(1);
+ yield* service.readSummary(input);
+ expect(fetches()).toBe(2);
+ }),
+ );
+
+ it.effect("does not cache failed transcript reads and reparses changed files", () =>
+ Effect.gen(function* () {
+ const now = yield* Clock.currentTimeMillis;
+ const { service, transcriptPath, fs } = yield* makeFixture({
+ diskRates: { fetchedAtMs: now, document: rateDocument },
+ });
+ const reads = vi.spyOn(TranscriptReader, "readTranscriptRecords").mockResolvedValueOnce(null);
+ const failed = yield* service.readSummary(input);
+ expect(failed.sources[0]!.scannedFiles).toBe(0);
+ const retried = yield* service.readSummary(input);
+ expect(retried.sources[0]!.scannedFiles).toBe(1);
+ yield* fs.writeFileString(
+ transcriptPath,
+ transcript + transcript.replace('"message"', '"second"'),
+ );
+ yield* service.readSummary(input);
+ expect(reads).toHaveBeenCalledTimes(3);
+ }),
+ );
+});
diff --git a/apps/server/src/usage/UsageService.ts b/apps/server/src/usage/UsageService.ts
index b4ff8e470..c7a8eed6c 100644
--- a/apps/server/src/usage/UsageService.ts
+++ b/apps/server/src/usage/UsageService.ts
@@ -26,12 +26,15 @@ import * as Cause from "effect/Cause";
import * as Clock from "effect/Clock";
import * as Context from "effect/Context";
import * as DateTime from "effect/DateTime";
+import * as Deferred from "effect/Deferred";
import * as Effect from "effect/Effect";
import * as FileSystem from "effect/FileSystem";
import * as Layer from "effect/Layer";
import * as Option from "effect/Option";
import * as Path from "effect/Path";
import * as Schema from "effect/Schema";
+import * as Scope from "effect/Scope";
+import * as Semaphore from "effect/Semaphore";
import { HttpClient, HttpClientResponse } from "effect/unstable/http";
import { ServerConfig } from "../config.ts";
@@ -61,6 +64,8 @@ const LITELLM_RATES_URL =
/** Rates move rarely; a day-old table keeps the page working offline. */
const RATES_TTL_MS = 24 * 60 * 60 * 1000;
+const RATES_FAILURE_BACKOFF_MS = 60_000;
+const RATES_MAX_FAILURE_BACKOFF_MS = 60 * 60 * 1000;
/**
* Files are filtered by mtime before opening. The slack covers a session whose
@@ -132,6 +137,30 @@ export const layerTestWithRates = (rateTable: RateTable) =>
export const layerTest = layerTestWithRates(new Map());
+// The service scope owns shared work; disconnecting one caller only cancels its wait.
+const singleFlight = <A, E>(scope: Scope.Scope) => {
+ const pending = new Map<string, Deferred.Deferred<A, E>>();
+ const acquire = Effect.fnUntraced(function* (key: string, work: Effect.Effect<A, E>) {
+ const existing = pending.get(key);
+ if (existing) return existing;
+ const shared = Deferred.makeUnsafe<A, E>();
+ pending.set(key, shared);
+ yield* Deferred.into(
+ work.pipe(
+ Effect.ensuring(
+ Effect.sync(() => {
+ pending.delete(key);
+ }),
+ ),
+ ),
+ shared,
+ ).pipe(Effect.forkIn(scope));
+ return shared;
+ }, Effect.uninterruptible);
+ return (key: string, work: Effect.Effect<A, E>) =>
+ acquire(key, work).pipe(Effect.flatMap(Deferred.await));
+};
+
export const make = Effect.gen(function* () {
const fileSystem = yield* FileSystem.FileSystem;
const path = yield* Path.Path;
@@ -140,8 +169,14 @@ export const make = Effect.gen(function* () {
const httpClient = yield* HttpClient.HttpClient;
const subscriptionAuth = SubscriptionAuthService.forSecretsDir(config.secretsDir);
+ const scope = yield* Scope.Scope;
const fileCache: ScanCache = new Map();
- let cacheDirty = false;
+ const shareSummary = singleFlight<UsageSummary, UsageReadError>(scope);
+ const shareFile = singleFlight<readonly UsageRecord[], never>(scope);
+ const shareRates = singleFlight<void, never>(scope);
+ const persistLock = yield* Semaphore.make(1);
+ let cacheRevision = 0;
+ let persistedRevision = 0;
const ratesCachePath = path.join(config.stateDir, "usage-model-rates.json");
const scanCachePath = path.join(config.stateDir, "usage-scan-cache.json");
@@ -149,56 +184,87 @@ export const make = Effect.gen(function* () {
let ratesFetchedAtMs: number | null = null;
let ratesStatus: UsageSummary["pricing"]["status"] = "unavailable";
- /**
- * Loads the LiteLLM rate table, preferring a fresh copy and falling back to
- * the on-disk snapshot. With neither, every model reports as unpriced rather
- * than the page failing.
- */
- const ensureRates = Effect.fn("UsageService.ensureRates")(function* () {
- const now = yield* Clock.currentTimeMillis;
- if (ratesFetchedAtMs !== null && now - ratesFetchedAtMs < RATES_TTL_MS) return;
+ let ratesNextAttemptAtMs = 0;
+ let ratesFailures = 0;
+ let ratesRefreshRunning = false;
- if (ratesFetchedAtMs === null) {
+ const loadRates = yield* Effect.cached(
+ Effect.gen(function* () {
const fromDisk = yield* fileSystem.readFileString(ratesCachePath).pipe(
- Effect.flatMap((raw) => decodeRatesCache(raw)),
+ Effect.flatMap(decodeRatesCache),
Effect.catchCause(() => Effect.succeed(null)),
);
- if (fromDisk !== null) {
- const parsed = parseRateTable(fromDisk.document);
- if (parsed.size > 0) {
- rates = parsed;
- ratesFetchedAtMs = fromDisk.fetchedAtMs;
- ratesStatus = "cached";
- if (now - fromDisk.fetchedAtMs < RATES_TTL_MS) return;
- }
+ if (fromDisk === null) return;
+ const parsed = parseRateTable(fromDisk.document);
+ if (parsed.size > 0) {
+ rates = parsed;
+ ratesFetchedAtMs = fromDisk.fetchedAtMs;
+ ratesStatus = "cached";
}
- }
-
- const fetched = yield* httpClient.get(LITELLM_RATES_URL).pipe(
- Effect.flatMap(HttpClientResponse.filterStatusOk),
- Effect.flatMap((response) => response.json),
- Effect.timeout(10_000),
- Effect.catchCause(() => Effect.succeed(null)),
- );
- if (fetched === null) {
- // The refresh failed; whatever we are serving is now past its TTL and
- // must not keep claiming to be fresh.
- if (rates.size > 0) ratesStatus = "cached";
- return;
- }
-
- const parsed = parseRateTable(fetched);
- if (parsed.size === 0) return;
+ }),
+ );
- rates = parsed;
- ratesFetchedAtMs = now;
- ratesStatus = "fresh";
+ const refreshRates = yield* Effect.cachedWithTTL(
+ Effect.gen(function* () {
+ const now = yield* Clock.currentTimeMillis;
+ if (now < ratesNextAttemptAtMs) return;
+ if (ratesFetchedAtMs !== null && now - ratesFetchedAtMs < RATES_TTL_MS) return;
+ const fetched = yield* httpClient.get(LITELLM_RATES_URL).pipe(
+ Effect.flatMap(HttpClientResponse.filterStatusOk),
+ Effect.flatMap((response) => response.json),
+ Effect.timeout(10_000),
+ Effect.catchCause(() => Effect.succeed(null)),
+ );
+ const parsed = parseRateTable(fetched);
+ const completedAt = yield* Clock.currentTimeMillis;
+ if (parsed.size === 0) {
+ ratesStatus = rates.size > 0 ? "cached" : "unavailable";
+ ratesNextAttemptAtMs =
+ completedAt +
+ Math.min(
+ RATES_MAX_FAILURE_BACKOFF_MS,
+ RATES_FAILURE_BACKOFF_MS * 2 ** Math.min(ratesFailures, 6),
+ );
+ ratesFailures += 1;
+ return;
+ }
+ rates = parsed;
+ ratesFetchedAtMs = completedAt;
+ ratesStatus = "fresh";
+ ratesFailures = 0;
+ ratesNextAttemptAtMs = 0;
+ yield* encodeRatesCache({ fetchedAtMs: completedAt, document: fetched }).pipe(
+ Effect.flatMap((serialized) => fileSystem.writeFileString(ratesCachePath, serialized)),
+ Effect.catchCause(() => Effect.void),
+ );
+ }),
+ 0,
+ );
- yield* encodeRatesCache({ fetchedAtMs: now, document: fetched }).pipe(
- Effect.flatMap((serialized) => fileSystem.writeFileString(ratesCachePath, serialized)),
- Effect.catchCause(() => Effect.void),
- );
- });
+ // Cold readers share the fetch; stale readers keep working while one scoped refresh runs.
+ const ensureRates = Effect.fn("UsageService.ensureRates")(
+ function* () {
+ yield* loadRates;
+ const now = yield* Clock.currentTimeMillis;
+ if (ratesFetchedAtMs !== null && now - ratesFetchedAtMs < RATES_TTL_MS) return;
+ ratesStatus = rates.size > 0 ? "cached" : "unavailable";
+ if (now < ratesNextAttemptAtMs) return;
+ if (rates.size === 0) {
+ yield* refreshRates;
+ } else if (!ratesRefreshRunning) {
+ ratesRefreshRunning = true;
+ yield* refreshRates.pipe(
+ Effect.ensuring(
+ Effect.sync(() => {
+ ratesRefreshRunning = false;
+ }),
+ ),
+ Effect.forkIn(scope),
+ );
+ }
+ },
+ (effect) => shareRates("rates", effect),
+ );
/**
* Claude's config dir is the home itself when overridden, but a default
@@ -264,18 +330,18 @@ export const make = Effect.gen(function* () {
);
const persistScanCache = Effect.fn("UsageService.persistScanCache")(function* () {
- if (!cacheDirty) return;
- // Cleared only after the write lands, so a failed persist is retried on
- // the next scan instead of leaving disk permanently stale.
+ if (cacheRevision === persistedRevision) return;
+ const revision = cacheRevision;
+ // Only acknowledge this revision; another scan can change the cache during the write.
yield* encodeScanCacheFile(encodeScanCache(fileCache)).pipe(
Effect.flatMap((serialized) => fileSystem.writeFileString(scanCachePath, serialized)),
Effect.map(() => {
- cacheDirty = false;
+ persistedRevision = revision;
}),
// A cache we cannot write is a slower next start, not a failed read.
Effect.catchCause(() => Effect.void),
);
- });
+ }, persistLock.withPermit);
/** Parses one transcript, reusing the cached result when it is unchanged. */
const readFileRecords = (
@@ -284,31 +350,36 @@ export const make = Effect.gen(function* () {
mtimeMs: number,
provider: UsageProviderKind,
): Effect.Effect<readonly UsageRecord[]> =>
- Effect.gen(function* () {
- const cached = fileCache.get(filePath);
- // Provider is part of the identity: if both providers were ever pointed
- // at one directory, a hit parsed by the other parser must not be reused.
- if (
- cached &&
- cached.size === size &&
- cached.mtimeMs === mtimeMs &&
- cached.provider === provider
- ) {
- return cached.records;
- }
+ shareFile(
+ JSON.stringify([filePath, size, mtimeMs, provider]),
+ Effect.gen(function* () {
+ const cached = fileCache.get(filePath);
+ // Provider is part of the identity: if both providers were ever pointed
+ // at one directory, a hit parsed by the other parser must not be reused.
+ if (
+ cached &&
+ cached.size === size &&
+ cached.mtimeMs === mtimeMs &&
+ cached.provider === provider
+ ) {
+ return cached.records;
+ }
- const parsed = yield* Effect.promise(() => readTranscriptRecords(filePath, provider));
- // A read failure is not an empty transcript: caching it under this
- // (size, mtime) would silently drop the file's usage until it changes.
- if (parsed === null) return [];
- // Stored already de-duplicated within the file, which is 99% of all
- // duplicates. The aggregator still runs the cross-file dedupe pass.
- const records = dedupeWithinFile(parsed);
-
- fileCache.set(filePath, { size, mtimeMs, provider, records });
- cacheDirty = true;
- return records;
- });
+ const parsed = yield* Effect.promise(() =>
+ readTranscriptRecords(filePath, provider, { size, mtimeMs }),
+ );
+ // A read failure is not an empty transcript: caching it under this
+ // (size, mtime) would silently drop the file's usage until it changes.
+ if (parsed === null) return [];
+ // Stored already de-duplicated within the file, which is 99% of all
+ // duplicates. The aggregator still runs the cross-file dedupe pass.
+ const records = dedupeWithinFile(parsed);
+
+ fileCache.set(filePath, { size, mtimeMs, provider, records });
+ cacheRevision += 1;
+ return records;
+ }),
+ );
const priceStepUsage = Effect.fn("UsageService.priceStepUsage")(function* (
input: PriceStepUsageInput,
@@ -367,6 +438,15 @@ export const make = Effect.gen(function* () {
const windowStartMs =
(hourlyWindow?.sinceTimeMs ?? DateTime.toEpochMillis(windowStart.value)) - MTIME_SLACK_MS;
+ const pricing = {
+ status: ratesStatus,
+ source: LITELLM_RATES_URL,
+ fetchedAt:
+ ratesFetchedAtMs === null
+ ? null
+ : DateTime.formatIso(DateTime.makeUnsafe(ratesFetchedAtMs)),
+ knownModels: rates.size,
+ };
const aggregator = new UsageAggregator({
timeZone: input.timeZone,
sinceDay: input.sinceDay,
@@ -441,7 +521,7 @@ export const make = Effect.gen(function* () {
windowStartMs,
retentionCutoffMs: startedAtMs - CACHE_RETENTION_DAYS * 24 * 60 * 60 * 1000,
});
- if (pruned > 0) cacheDirty = true;
+ if (pruned > 0) cacheRevision += 1;
yield* persistScanCache();
const aggregated = aggregator.finish();
@@ -461,20 +541,26 @@ export const make = Effect.gen(function* () {
buckets: aggregated.buckets,
sources,
planLimits,
- pricing: {
- status: ratesStatus,
- source: LITELLM_RATES_URL,
- fetchedAt:
- ratesFetchedAtMs === null
- ? null
- : DateTime.formatIso(DateTime.makeUnsafe(ratesFetchedAtMs)),
- knownModels: rates.size,
- },
+ pricing,
scanDurationMs: Math.max(0, finishedAtMs - startedAtMs),
} satisfies UsageSummary;
});
- return { readSummary, priceStepUsage } as const;
+ return {
+ readSummary: (input: UsageSummaryInput) =>
+ shareSummary(
+ JSON.stringify([
+ input.sinceDay,
+ input.untilDay,
+ input.timeZone,
+ input.resolution ?? "day",
+ input.sinceTime,
+ input.untilTime,
+ ]),
+ readSummary(input),
+ ),
+ priceStepUsage,
+ } as const;
});
export const layer = Layer.effect(UsageService, make);
diff --git a/apps/server/src/usage/usageTranscriptReader.test.ts b/apps/server/src/usage/usageTranscriptReader.test.ts
new file mode 100644
index 000000000..25041e300
--- /dev/null
+++ b/apps/server/src/usage/usageTranscriptReader.test.ts
@@ -0,0 +1,89 @@
+// @effect-diagnostics nodeBuiltinImport:off
+import * as NodeFSP from "node:fs/promises";
+import * as NodeOS from "node:os";
+import * as NodePath from "node:path";
+import { afterEach, describe, expect, it, vi } from "vite-plus/test";
+
+import { listTranscriptFiles, readTranscriptRecords } from "./usageTranscriptReader.ts";
+
+vi.mock("node:fs/promises", async (importOriginal) => {
+ const original = await importOriginal<typeof NodeFSP>();
+ return { ...original, readdir: vi.fn(original.readdir) };
+});
+
+const directories: string[] = [];
+const tempDirectory = async () => {
+ const directory = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "akeru-usage-reader-"));
+ directories.push(directory);
+ return directory;
+};
+const line = JSON.stringify({
+ type: "assistant",
+ timestamp: "2026-09-07T04:05:13.944Z",
+ sessionId: "s",
+ message: { id: "m", model: "claude-fable-5", usage: { input_tokens: 1, output_tokens: 2 } },
+});
+
+afterEach(async () => {
+ vi.restoreAllMocks();
+ for (const directory of directories.splice(0))
+ await NodeFSP.rm(directory, { recursive: true, force: true });
+});
+
+describe("usage transcript single-flight", () => {
+ it("shares directory walks across time windows, not the filtered results", async () => {
+ const directory = await tempDirectory();
+ const file = NodePath.join(directory, "session.jsonl");
+ await NodeFSP.writeFile(file, line);
+ const readdir = vi.mocked(NodeFSP.readdir);
+ readdir.mockClear();
+ const [all, none] = await Promise.all([
+ listTranscriptFiles(directory, 0),
+ listTranscriptFiles(directory, Infinity),
+ ]);
+ expect(all.map((entry) => entry.path)).toEqual([file]);
+ expect(none).toEqual([]);
+ expect(readdir).toHaveBeenCalledTimes(1);
+ await listTranscriptFiles(directory, 0);
+ expect(readdir).toHaveBeenCalledTimes(2);
+ });
+
+ it("shares only simultaneous reads for the same provider and file version", async () => {
+ const file = NodePath.join(await tempDirectory(), "session.jsonl");
+ await NodeFSP.writeFile(file, line + "\n");
+ const version = { size: Buffer.byteLength(line) + 1, mtimeMs: 1 };
+ const first = readTranscriptRecords(file, "claude", version);
+ expect(readTranscriptRecords(file, "claude", version)).toBe(first);
+ const otherProvider = readTranscriptRecords(file, "codex", version);
+ const otherVersion = readTranscriptRecords(file, "claude", { ...version, mtimeMs: 2 });
+ expect(otherProvider).not.toBe(first);
+ expect(otherVersion).not.toBe(first);
+ expect((await first)?.length).toBe(1);
+ await Promise.all([otherProvider, otherVersion]);
+ const later = readTranscriptRecords(file, "claude", version);
+ expect(later).not.toBe(first);
+ await later;
+ });
+
+ it("reparses partial final lines, truncations, and replacements without retaining stale parser state", async () => {
+ const file = NodePath.join(await tempDirectory(), "session.jsonl");
+ const read = async () => readTranscriptRecords(file, "claude", await NodeFSP.stat(file));
+ await NodeFSP.writeFile(file, line.slice(0, -4));
+ expect(await read()).toEqual([]);
+ await NodeFSP.appendFile(file, line.slice(-4));
+ expect((await read())?.map((record) => record.dedupeKey)).toEqual(["m:"]);
+ await NodeFSP.writeFile(file, "");
+ expect(await read()).toEqual([]);
+ await NodeFSP.writeFile(file, line.replace('"m"', '"replacement"') + "\n");
+ expect((await read())?.map((record) => record.dedupeKey)).toEqual(["replacement:"]);
+ });
+
+ it("shares read failures without retaining them after the file becomes readable", async () => {
+ const file = NodePath.join(await tempDirectory(), "missing.jsonl");
+ const first = readTranscriptRecords(file, "claude");
+ expect(readTranscriptRecords(file, "claude")).toBe(first);
+ expect(await first).toBeNull();
+ await NodeFSP.writeFile(file, line);
+ expect((await readTranscriptRecords(file, "claude"))?.length).toBe(1);
+ });
+});
diff --git a/apps/server/src/usage/usageTranscriptReader.ts b/apps/server/src/usage/usageTranscriptReader.ts
index c72f0c24d..316ef1a85 100644
--- a/apps/server/src/usage/usageTranscriptReader.ts
+++ b/apps/server/src/usage/usageTranscriptReader.ts
@@ -38,10 +38,22 @@ export interface TranscriptFile {
* removed while the walk is in flight, and a partial listing is far better than
* failing the page.
*/
-export async function listTranscriptFiles(
+const walks = new Map<string, Promise<readonly TranscriptFile[]>>();
+const reads = new Map<string, Promise<readonly UsageRecord[] | null>>();
+
+export function listTranscriptFiles(
root: string,
sinceMs: number,
): Promise<readonly TranscriptFile[]> {
+ let walk = walks.get(root);
+ if (!walk) {
+ walk = walkTranscriptFiles(root).finally(() => walks.delete(root));
+ walks.set(root, walk);
+ }
+ return walk.then((files) => files.filter((file) => file.mtimeMs >= sinceMs));
+}
+
+async function walkTranscriptFiles(root: string): Promise<readonly TranscriptFile[]> {
const found: TranscriptFile[] = [];
const walk = async (dir: string): Promise<void> => {
@@ -60,9 +72,7 @@ export async function listTranscriptFiles(
if (!entry.name.endsWith(".jsonl")) continue;
try {
const stats = await NodeFSP.stat(child);
- if (stats.mtimeMs >= sinceMs) {
- found.push({ path: child, size: stats.size, mtimeMs: stats.mtimeMs });
- }
+ found.push({ path: child, size: stats.size, mtimeMs: stats.mtimeMs });
} catch {
// Vanished between readdir and stat.
}
@@ -102,7 +112,21 @@ export async function readDirectoryVolumeId(path: string): Promise<string> {
* their own, so those still have to pass through the reducer to keep model
* attribution correct.
*/
-export async function readTranscriptRecords(
+export function readTranscriptRecords(
+ filePath: string,
+ provider: UsageProviderKind,
+ version?: Pick<TranscriptFile, "size" | "mtimeMs">,
+): Promise<readonly UsageRecord[] | null> {
+ const key = JSON.stringify([filePath, provider, version?.size, version?.mtimeMs]);
+ let read = reads.get(key);
+ if (!read) {
+ read = scanTranscriptRecords(filePath, provider).finally(() => reads.delete(key));
+ reads.set(key, read);
+ }
+ return read;
+}
+
+async function scanTranscriptRecords(
filePath: string,
provider: UsageProviderKind,
): Promise<readonly UsageRecord[] | null> {