Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.
perf(logging): drain bounded asynchronous writes before desktop exit (#203)
Leo9f35294eea5f
diff --git a/apps/desktop/src/app/DesktopApp.ts b/apps/desktop/src/app/DesktopApp.tsindex ecb3541e6..6fb9d666d 100644--- a/apps/desktop/src/app/DesktopApp.ts+++ b/apps/desktop/src/app/DesktopApp.ts@@ -324,7 +324,7 @@ const scopedProgram = Effect.scoped(yield* Effect.forEach(instances, (instance) => instance.stop(), {concurrency: "unbounded",});- }).pipe(Effect.ensuring(shutdown.markComplete)),+ }),);yield* startup;diff --git a/apps/desktop/src/app/DesktopLifecycle.test.ts b/apps/desktop/src/app/DesktopLifecycle.test.tsindex 66f0eaa8e..5cf74a9be 100644--- a/apps/desktop/src/app/DesktopLifecycle.test.ts+++ b/apps/desktop/src/app/DesktopLifecycle.test.ts@@ -1,6 +1,9 @@import { assert, describe, it } from "@effect/vitest";import * as Deferred from "effect/Deferred";import * as Effect from "effect/Effect";+import * as Fiber from "effect/Fiber";++import { DesktopTraceShutdown } from "./DesktopObservability.ts";import * as Layer from "effect/Layer";import * as Ref from "effect/Ref";@@ -145,7 +148,7 @@ describe("DesktopLifecycle", () => {});}- it.effect("destroys windows before waiting for backend shutdown", () =>+ it.effect("keeps windows alive until shutdown acknowledgement", () =>Effect.gen(function* () {const appListeners = new Map<string, (...args: readonly unknown[]) => void>();const shutdownRequested = yield* Deferred.make<void>();@@ -201,13 +204,170 @@ describe("DesktopLifecycle", () => {yield* Deferred.succeed(allowShutdown, undefined);yield* Deferred.await(quitRequested);- assert.deepEqual(eventsBeforeCleanup, ["flush", "destroy", "request"]);- assert.deepEqual(events, ["flush", "destroy", "request", "quit"]);+ assert.deepEqual(eventsBeforeCleanup, ["flush", "request"]);+ assert.deepEqual(events, ["flush", "request", "destroy", "quit"]);+ }),+ ).pipe(Effect.provide(layer));+ }),+ );++ it.effect("allows native quit after shutdown setup fails", () =>+ Effect.gen(function* () {+ const appListeners = new Map<string, (...args: readonly unknown[]) => void>();+ const quitRequested = yield* Deferred.make<void>();+ const quit = Deferred.succeed(quitRequested, undefined).pipe(Effect.asVoid);+ const layer = DesktopLifecycle.layer.pipe(+ Layer.provideMerge(makeElectronAppLayer(appListeners, quit)),+ Layer.provideMerge(electronThemeLayer),+ Layer.provideMerge(makeElectronWindowLayer()),+ Layer.provideMerge(+ makeDesktopWindowLayer({+ flushMainWindowBounds: Effect.die("bounds write failed"),+ }),+ ),+ Layer.provideMerge(+ Layer.succeed(DesktopEnvironment.DesktopEnvironment, {+ platform: "darwin",+ isDevelopment: false,+ } as DesktopEnvironment.DesktopEnvironment["Service"]),+ ),+ Layer.provideMerge(DesktopShutdown.layer),+ Layer.provideMerge(DesktopState.layer),+ );++ yield* Effect.scoped(+ Effect.gen(function* () {+ const lifecycle = yield* DesktopLifecycle.DesktopLifecycle;+ yield* lifecycle.register;+ let prevented = false;+ appListeners.get("before-quit")?.({+ preventDefault: () => {+ prevented = true;+ },+ });+ yield* Deferred.await(quitRequested);+ assert.isTrue(prevented);++ let retryPrevented = false;+ appListeners.get("before-quit")?.({+ preventDefault: () => {+ retryPrevented = true;+ },+ });+ assert.isFalse(retryPrevented);}),).pipe(Effect.provide(layer));}),);+ for (const destroyFails of [false, true]) {+ it.effect(+ `completes nested app shutdown before native quit (destroyFails=${destroyFails})`,+ () =>+ Effect.gen(function* () {+ const appListeners = new Map<string, (...args: readonly unknown[]) => void>();+ const registered = yield* Deferred.make<void>();+ const boundsEntered = yield* Deferred.make<void>();+ const allowBounds = yield* Deferred.make<void>();+ const stopEntered = yield* Deferred.make<void>();+ const allowStop = yield* Deferred.make<void>();+ const closeEntered = yield* Deferred.make<void>();+ const allowClose = yield* Deferred.make<void>();+ const nativeQuit = yield* Deferred.make<void>();+ const events: string[] = [];+ const quit = Effect.sync(() => {+ events.push("native-quit");+ }).pipe(Effect.andThen(Deferred.succeed(nativeQuit, undefined)), Effect.asVoid);+ const layer = Layer.mergeAll(+ DesktopLifecycle.layer,+ makeElectronAppLayer(appListeners, quit),+ electronThemeLayer,+ makeElectronWindowLayer(+ Effect.sync(() => {+ events.push("destroy");+ if (destroyFails) throw new Error("invalid guest");+ }),+ ),+ makeDesktopWindowLayer({+ flushMainWindowBounds: Effect.gen(function* () {+ events.push("bounds");+ yield* Deferred.succeed(boundsEntered, undefined);+ yield* Deferred.await(allowBounds);+ }),+ }),+ Layer.succeed(DesktopEnvironment.DesktopEnvironment, {+ platform: "linux",+ isDevelopment: false,+ } as DesktopEnvironment.DesktopEnvironment["Service"]),+ DesktopShutdown.layer,+ DesktopState.layer,+ Layer.succeed(DesktopTraceShutdown, {+ close: Effect.gen(function* () {+ events.push("trace-close");+ assert.isFalse(appListeners.has("before-quit"));+ yield* Deferred.succeed(closeEntered, undefined);+ yield* Deferred.await(allowClose);+ events.push("trace-ack");+ }),+ }),+ Layer.effectDiscard(Effect.addFinalizer(() => Deferred.await(nativeQuit))),+ );+ const main = yield* Effect.scoped(+ Effect.gen(function* () {+ const shutdown = yield* DesktopShutdown.DesktopShutdown;+ yield* Effect.addFinalizer(() =>+ Effect.gen(function* () {+ events.push("backend-stop");+ yield* Deferred.succeed(stopEntered, undefined);+ yield* Deferred.await(allowStop);+ }),+ );+ const lifecycle = yield* DesktopLifecycle.DesktopLifecycle;+ yield* lifecycle.register;+ yield* Deferred.succeed(registered, undefined);+ yield* shutdown.awaitRequest;+ }),+ ).pipe(+ Effect.withSpan("desktop.app"),+ Effect.ensuring(DesktopShutdown.acknowledgeShutdown),+ Effect.provide(layer),+ Effect.forkChild,+ );+ yield* Deferred.await(registered);+ let prevented = false;+ appListeners.get("before-quit")?.({+ preventDefault: () => {+ prevented = true;+ },+ });+ yield* Deferred.await(boundsEntered);+ assert.isTrue(prevented);+ assert.isFalse(yield* Deferred.isDone(stopEntered));+ yield* Deferred.succeed(allowBounds, undefined);+ yield* Deferred.await(stopEntered);+ const beforeBackendAcknowledgement = [...events];+ assert.isFalse(yield* Deferred.isDone(closeEntered));+ yield* Deferred.succeed(allowStop, undefined);+ yield* Deferred.await(closeEntered);+ const beforeTraceAcknowledgement = [...events];+ assert.isFalse(yield* Deferred.isDone(nativeQuit));+ yield* Deferred.succeed(allowClose, undefined);+ yield* Deferred.await(nativeQuit);+ yield* Fiber.join(main);+ assert.deepEqual(beforeBackendAcknowledgement, ["bounds", "backend-stop"]);+ assert.deepEqual(beforeTraceAcknowledgement, ["bounds", "backend-stop", "trace-close"]);+ assert.deepEqual(events, [+ "bounds",+ "backend-stop",+ "trace-close",+ "trace-ack",+ "destroy",+ "native-quit",+ ]);+ }),+ );+ }+it.effect("ignores app activation while quitting", () =>Effect.gen(function* () {const appListeners = new Map<string, (...args: readonly unknown[]) => void>();diff --git a/apps/desktop/src/app/DesktopLifecycle.ts b/apps/desktop/src/app/DesktopLifecycle.tsindex 6a98e59eb..f1c3d8529 100644--- a/apps/desktop/src/app/DesktopLifecycle.ts+++ b/apps/desktop/src/app/DesktopLifecycle.ts@@ -82,13 +82,14 @@ function addScopedListener<Args extends ReadonlyArray<unknown>>(}const requestDesktopShutdownAndWait = Effect.fn("desktop.lifecycle.requestShutdownAndWait")(- function* (- afterBoundsFlush: Effect.Effect<void> = Effect.void,- ): Effect.fn.Return<void, never, DesktopShutdown.DesktopShutdown | DesktopWindow.DesktopWindow> {+ function* (): Effect.fn.Return<+ void,+ never,+ DesktopShutdown.DesktopShutdown | DesktopWindow.DesktopWindow+ > {const shutdown = yield* DesktopShutdown.DesktopShutdown;const desktopWindow = yield* DesktopWindow.DesktopWindow;yield* desktopWindow.flushMainWindowBounds;- yield* afterBoundsFlush;yield* shutdown.request;yield* shutdown.awaitComplete;},@@ -120,23 +121,36 @@ function handleBeforeQuit(const electronWindow = yield* ElectronWindow.ElectronWindow;yield* Ref.set(state.quitting, true);yield* logLifecycleInfo("before-quit received");- yield* requestDesktopShutdownAndWait(- electronWindow.destroyAll.pipe(- Effect.catchCause((cause) =>- logLifecycleError("failed to destroy windows before shutdown", { cause }),- ),+ // Keep Electron windows alive until backend and trace I/O have drained.+ yield* requestDesktopShutdownAndWait();+ yield* electronWindow.destroyAll.pipe(+ Effect.catchCause((cause) =>+ logLifecycleError("failed to destroy windows after shutdown", { cause }),),);}).pipe(Effect.withSpan("desktop.lifecycle.beforeQuit")),- ).finally(() => {- markQuitAllowed();- void runEffect(- Effect.gen(function* () {- const electronApp = yield* ElectronApp.ElectronApp;- yield* electronApp.quit;- }).pipe(Effect.withSpan("desktop.lifecycle.quitAfterShutdown")),- );- });+ )+ .then(() => {+ markQuitAllowed();+ void runEffect(+ Effect.gen(function* () {+ const electronApp = yield* ElectronApp.ElectronApp;+ yield* electronApp.quit;+ }).pipe(Effect.withSpan("desktop.lifecycle.quitAfterShutdown")),+ );+ })+ .catch((cause: unknown) => {+ // The original event was already prevented. Release the guard before+ // retrying native quit so a failed drain cannot strand the application.+ markQuitAllowed();+ void runEffect(+ Effect.gen(function* () {+ yield* logLifecycleError("desktop shutdown failed before quit", { cause });+ const electronApp = yield* ElectronApp.ElectronApp;+ yield* electronApp.quit;+ }).pipe(Effect.withSpan("desktop.lifecycle.quitAfterShutdownFailure")),+ );+ });}function quitFromSignal(diff --git a/apps/desktop/src/app/DesktopObservability.test.ts b/apps/desktop/src/app/DesktopObservability.test.tsindex cd90e7951..1702bf9f3 100644--- a/apps/desktop/src/app/DesktopObservability.test.ts+++ b/apps/desktop/src/app/DesktopObservability.test.ts@@ -3,12 +3,14 @@ import * as NodeServices from "@effect/platform-node/NodeServices";import { assert, describe, it } from "@effect/vitest";import * as Effect from "effect/Effect";import * as FileSystem from "effect/FileSystem";+import * as Fiber from "effect/Fiber";import * as Layer from "effect/Layer";import * as Schema from "effect/Schema";import * as DesktopConfig from "./DesktopConfig.ts";import * as DesktopEnvironment from "./DesktopEnvironment.ts";import * as DesktopObservability from "./DesktopObservability.ts";+import * as DesktopShutdown from "./DesktopShutdown.ts";const DesktopBackendChildLogRecord = Schema.Struct({message: Schema.String,@@ -133,6 +135,57 @@ describe("DesktopObservability", () => {),);+ it.effect("acknowledges the actual observability layer after the app span reaches disk", () =>+ Effect.gen(function* () {+ const fileSystem = yield* FileSystem.FileSystem;+ const baseDir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "desktop-quit-trace-" });+ const environmentLayer = makeEnvironmentLayer(baseDir);+ yield* Effect.gen(function* () {+ const environment = yield* DesktopEnvironment.DesktopEnvironment;+ const shutdown = yield* DesktopShutdown.DesktopShutdown;+ const nativeQuit = yield* shutdown.awaitComplete.pipe(+ Effect.andThen(+ Effect.gen(function* () {+ const text = yield* fileSystem.readFileString(+ environment.path.join(environment.logDir, "desktop.trace.ndjson"),+ );+ const records = text+ .trim()+ .split("\n")+ .map((line) => decodeTraceRecordLine(line));+ assert.isTrue(records.some((record) => record.name === "desktop.app"));+ assert.isTrue(records.some((record) => record.name === "desktop.backend.stop.test"));+ }),+ ),+ Effect.forkScoped,+ );+ yield* Effect.scoped(+ Effect.gen(function* () {+ yield* Effect.addFinalizer(() =>+ Effect.void.pipe(Effect.withSpan("desktop.backend.stop.test")),+ );+ yield* shutdown.request;+ yield* shutdown.awaitRequest;+ }),+ ).pipe(+ Effect.withSpan("desktop.app"),+ Effect.ensuring(DesktopShutdown.acknowledgeShutdown),+ );+ yield* Fiber.join(nativeQuit);+ }).pipe(+ Effect.provide(+ Layer.mergeAll(+ DesktopShutdown.layer,+ DesktopObservability.layer.pipe(Layer.provideMerge(environmentLayer)),+ ),+ ),+ );+ }).pipe(+ Effect.scoped,+ Effect.provide(Layer.mergeAll(NodeServices.layer, NodeHttpClient.layerUndici)),+ ),+ );+it.effect("buffers backend child output and persists it only when a failure is reported", () =>Effect.gen(function* () {const fileSystem = yield* FileSystem.FileSystem;diff --git a/apps/desktop/src/app/DesktopObservability.ts b/apps/desktop/src/app/DesktopObservability.tsindex c393f4ccd..48b17bfbe 100644--- a/apps/desktop/src/app/DesktopObservability.ts+++ b/apps/desktop/src/app/DesktopObservability.ts@@ -60,6 +60,11 @@ export class DesktopBackendOutputLogFactory extends Context.Service<}>()("@t3tools/desktop/app/DesktopObservability/DesktopBackendOutputLogFactory") {}+export class DesktopTraceShutdown extends Context.Service<+ DesktopTraceShutdown,+ { readonly close: Effect.Effect<void> }+>()("@t3tools/desktop/app/DesktopObservability/DesktopTraceShutdown") {}+const textEncoder = new TextEncoder();const textDecoder = new TextDecoder();@@ -601,7 +606,10 @@ const tracerLayer = Layer.unwrap(...(delegate ? { delegate } : {}),});- return Layer.succeed(Tracer.Tracer, tracer);+ return Layer.mergeAll(+ Layer.succeed(Tracer.Tracer, tracer),+ Layer.succeed(DesktopTraceShutdown, { close: sink.close() }),+ );}),).pipe(Layer.provide(OtlpExporter.layerFlusher), Layer.provideMerge(OtlpSerialization.layerJson));diff --git a/apps/desktop/src/app/DesktopShutdown.test.ts b/apps/desktop/src/app/DesktopShutdown.test.tsnew file mode 100644index 000000000..a66550eea--- /dev/null+++ b/apps/desktop/src/app/DesktopShutdown.test.ts@@ -0,0 +1,105 @@+import { afterEach, expect, vi } from "vite-plus/test";+import { it } from "@effect/vitest";+import * as Effect from "effect/Effect";+import * as Fiber from "effect/Fiber";+import * as FileSystem from "effect/FileSystem";+import * as Layer from "effect/Layer";+import * as Logger from "effect/Logger";+import * as Tracer from "effect/Tracer";+import * as NodeServices from "@effect/platform-node/NodeServices";+import { makeLocalFileTracer, makeTraceSink } from "@t3tools/shared/observability";+import { DesktopTraceShutdown } from "./DesktopObservability.ts";+import * as DesktopShutdown from "./DesktopShutdown.ts";++const writer = vi.hoisted(() => ({ append: vi.fn<(data: string) => Promise<void>>() }));+vi.mock("node:fs/promises", async (importOriginal) => {+ const original = await importOriginal<typeof import("node:fs/promises")>();+ return {+ ...original,+ appendFile: (_path: unknown, data: Uint8Array) => writer.append(Buffer.from(data).toString()),+ };+});+afterEach(() => vi.clearAllMocks());++for (const fail of [false, true]) {+ it.effect(+ `waits for real trace writer acknowledgement before fake native exit (failure=${fail})`,+ () => {+ const entered = Promise.withResolvers<void>();+ const release = Promise.withResolvers<void>();+ const events: string[] = [];+ const records: string[] = [];+ writer.append.mockImplementation(async (data) => {+ records.push(data);+ entered.resolve();+ await release.promise;+ events.push(fail ? "write-error" : "write-complete");+ if (fail) throw new Error("blocked writer failed");+ });+ return Effect.scoped(+ Effect.gen(function* () {+ const fs = yield* FileSystem.FileSystem;+ const directory = yield* fs.makeTempDirectoryScoped();+ const sink = yield* makeTraceSink({+ filePath: `${directory}/desktop.trace.ndjson`,+ maxBytes: 100_000,+ maxFiles: 1,+ batchWindowMs: 60_000,+ });+ const tracer = yield* makeLocalFileTracer({+ filePath: sink.filePath,+ maxBytes: 100_000,+ maxFiles: 1,+ batchWindowMs: 60_000,+ sink,+ });+ const shutdown = yield* DesktopShutdown.DesktopShutdown;+ const exit = yield* shutdown.awaitComplete.pipe(+ Effect.andThen(+ Effect.sync(() => {+ events.push("native-exit");+ }),+ ),+ Effect.forkChild,+ );+ const program = yield* Effect.void.pipe(+ Effect.withSpan("desktop.app"),+ Effect.provideService(Tracer.Tracer, tracer),+ Effect.ensuring(+ DesktopShutdown.acknowledgeShutdown.pipe(+ Effect.provideService(DesktopTraceShutdown, { close: sink.close() }),+ ),+ ),+ Effect.forkChild,+ );+ yield* Effect.promise(() => entered.promise);+ expect(yield* shutdown.isComplete).toBe(false);+ expect(events).toEqual([]);+ release.resolve();+ yield* Fiber.join(program);+ yield* Fiber.join(exit);+ expect(yield* shutdown.isComplete).toBe(true);+ expect(records.join("")).toContain('"name":"desktop.app"');+ expect(events).toEqual(+ fail+ ? ["write-error", "error-reported", "native-exit"]+ : ["write-complete", "native-exit"],+ );+ }),+ ).pipe(+ Effect.provide(+ Layer.mergeAll(+ DesktopShutdown.layer,+ NodeServices.layer,+ Logger.layer([+ Logger.make(({ message }) => {+ if (String(message).includes("could not be persisted"))+ events.push("error-reported");+ }),+ ]),+ ),+ ),+ );+ },+ );+}diff --git a/apps/desktop/src/app/DesktopShutdown.ts b/apps/desktop/src/app/DesktopShutdown.tsindex 78b77b565..f349cce8f 100644--- a/apps/desktop/src/app/DesktopShutdown.ts+++ b/apps/desktop/src/app/DesktopShutdown.ts@@ -4,6 +4,8 @@ import * as Effect from "effect/Effect";import * as Layer from "effect/Layer";import * as Ref from "effect/Ref";+import { DesktopTraceShutdown } from "./DesktopObservability.ts";+export class DesktopShutdown extends Context.Service<DesktopShutdown,{@@ -32,4 +34,12 @@ const make = Effect.gen(function* () {});});+// Run outside the app span so its final record is included in the drain.+export const acknowledgeShutdown = Effect.gen(function* () {+ const trace = yield* DesktopTraceShutdown;+ const shutdown = yield* DesktopShutdown;+ yield* trace.close;+ yield* shutdown.markComplete;+}).pipe(Effect.uninterruptible, Effect.withTracerEnabled(false));+export const layer = Layer.effect(DesktopShutdown, make);diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.tsindex 1ef71a126..f6bd1731f 100644--- a/apps/desktop/src/main.ts+++ b/apps/desktop/src/main.ts@@ -204,4 +204,8 @@ const desktopRuntimeLayer = desktopApplicationRuntimeLayer.pipe(Layer.provideMerge(DesktopPreReadyPlatform.layer),);-DesktopApp.program.pipe(Effect.provide(desktopRuntimeLayer), NodeRuntime.runMain);+DesktopApp.program.pipe(+ Effect.ensuring(DesktopShutdown.acknowledgeShutdown),+ Effect.provide(desktopRuntimeLayer),+ NodeRuntime.runMain,+);diff --git a/apps/server/src/provider/Layers/EventNdjsonLogger.test.ts b/apps/server/src/provider/Layers/EventNdjsonLogger.test.tsindex f6fb557e4..dcea46a50 100644--- a/apps/server/src/provider/Layers/EventNdjsonLogger.test.ts+++ b/apps/server/src/provider/Layers/EventNdjsonLogger.test.ts@@ -4,8 +4,9 @@ import * as NodeOS from "node:os";import * as NodePath from "node:path";import { ThreadId } from "@t3tools/contracts";-import { assert, describe, it } from "@effect/vitest";+import { assert, describe, expect, it } from "@effect/vitest";import * as Clock from "effect/Clock";+import * as Deferred from "effect/Deferred";import * as Effect from "effect/Effect";import * as Fiber from "effect/Fiber";import * as Logger from "effect/Logger";@@ -81,6 +82,35 @@ describe("EventNdjsonLogger", () => {}).pipe(Effect.provide(Logger.layer([logCapture], { mergeWithExisting: false })));});+ it.effect("closes empty and buffered stores when the owning fiber is interrupted", () =>+ Effect.gen(function* () {+ const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-interrupt-"));+ try {+ for (const count of [0, 1]) {+ const basePath = NodePath.join(tempDir, `provider-${count}.ndjson`);+ const ready = yield* Deferred.make<void>();+ const owner = yield* Effect.gen(function* () {+ const store = yield* makeEventNdjsonLogStore(basePath, { batchWindowMs: 10_000 });+ yield* Effect.addFinalizer(() => store.close());+ if (count > 0) {+ yield* store.logger("native").write({ id: "accepted" }, ThreadId.make("thread-1"));+ }+ yield* Deferred.succeed(ready, undefined);+ return yield* Effect.never;+ }).pipe(Effect.scoped, Effect.forkChild);+ yield* Deferred.await(ready);+ yield* Fiber.interrupt(owner);+ if (count > 0) {+ const line = NodeFS.readFileSync(ownedLogPath(basePath, "thread-1"), "utf8").trim();+ assert.equal(parseLogLine(line).payload, '{"id":"accepted"}');+ }+ }+ } finally {+ NodeFS.rmSync(tempDir, { recursive: true, force: true });+ }+ }),+ );+it.effect("writes effect-style lines to thread-scoped files", () =>Effect.gen(function* () {const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-"));@@ -251,6 +281,7 @@ describe("EventNdjsonLogger", () => {assert.equal(NodeFS.existsSync(threadPath), false);yield* TestClock.adjust(1_000);+ yield* store.flush;assert.equal(NodeFS.existsSync(threadPath), true);yield* store.close();} finally {@@ -276,6 +307,7 @@ describe("EventNdjsonLogger", () => {yield* logger.write({ id: "accepted" }, ThreadId.make("thread-interrupted"));yield* TestClock.adjust(1_000);+ yield* store.flush;assert.equal(NodeFS.existsSync(threadPath), true);assert.include(NodeFS.readFileSync(threadPath, "utf8"), '{"id":"accepted"}');@@ -521,7 +553,7 @@ describe("EventNdjsonLogger", () => {}),);- it("attributes batches that were written before a later chunk fails", () => {+ it("attributes batches that were written before a later chunk fails", async () => {const records: ReadonlyArray<PendingRecord> = [{stream: "native",@@ -539,10 +571,10 @@ describe("EventNdjsonLogger", () => {const attributed: Array<PendingRecord> = [];let writes = 0;- assert.throws(() =>+ await expect(writeBatchedMessages({- write: () => {+ write: async () => {writes += 1;if (writes === 2) throw new Error("simulated disk exhaustion");},@@ -551,10 +583,93 @@ describe("EventNdjsonLogger", () => {5,(written) => attributed.push(...written),),- );+ ).rejects.toThrow("simulated disk exhaustion");assert.deepEqual(attributed, [records[0]]);});+ it.effect(+ "backpressures concurrent batches within byte and record limits and flushes in order",+ () =>+ Effect.gen(function* () {+ const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-bound-"));+ const basePath = NodePath.join(tempDir, "events.log");+ try {+ const store = yield* makeEventNdjsonLogStore(basePath, {+ maxBufferedBytes: 160,+ maxBufferedRecords: 2,+ batchWindowMs: 1_000,+ });+ const native = store.logger("native");+ const canonical = store.logger("canonical");+ yield* Effect.all(+ Array.from({ length: 20 }, (_, index) =>+ (index % 2 ? canonical : native).write({ id: index }, ThreadId.make("ordered")),+ ),+ { concurrency: "unbounded" },+ );+ yield* native.close();+ yield* store.flush;+ yield* store.close();+ const lines = NodeFS.readFileSync(ownedLogPath(basePath, "ordered"), "utf8")+ .trim()+ .split("\n")+ .map(parseLogLine);+ assert.deepEqual(+ lines.map((line) => line.payload),+ Array.from({ length: 20 }, (_, id) => encodeUnknownJson({ id })),+ );+ let serializedAfterClose = false;+ yield* native.write(+ {+ toJSON: () => {+ serializedAfterClose = true;+ return {};+ },+ },+ null,+ );+ assert.isFalse(serializedAfterClose);+ } finally {+ NodeFS.rmSync(tempDir, { recursive: true, force: true });+ }+ }),+ );++ it.effect(+ "reports rejected oversized records and asynchronous disk failures without failing provider work",+ () =>+ Effect.gen(function* () {+ const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-error-"));+ const basePath = NodePath.join(tempDir, "events.log");+ const messages: unknown[] = [];+ const capture = Logger.make<unknown, void>(({ message }) => {+ messages.push(message);+ });+ try {+ yield* Effect.gen(function* () {+ const store = yield* makeEventNdjsonLogStore(basePath, {+ maxBufferedBytes: 256,+ batchWindowMs: 0,+ });+ NodeFS.mkdirSync(ownedLogPath(basePath, "broken"));+ const logger = store.logger("native");+ yield* logger.write({ payload: "x".repeat(1024) }, ThreadId.make("oversized"));+ yield* logger.write({ id: "failed" }, ThreadId.make("broken"));+ yield* logger.write({ id: "retained" }, ThreadId.make("healthy"));+ yield* store.close();+ }).pipe(Effect.provide(Logger.layer([capture], { mergeWithExisting: false })));+ assert.isFalse(NodeFS.existsSync(ownedLogPath(basePath, "oversized")));+ assert.include(+ NodeFS.readFileSync(ownedLogPath(basePath, "healthy"), "utf8"),+ "retained",+ );+ assert.equal(messages.length, 2);+ } finally {+ NodeFS.rmSync(tempDir, { recursive: true, force: true });+ }+ }),+ );+it.effect("reports logical provider log writes to resource attribution", () =>Effect.gen(function* () {const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-"));diff --git a/apps/server/src/provider/Layers/EventNdjsonLogger.ts b/apps/server/src/provider/Layers/EventNdjsonLogger.tsindex e07121ea7..6e1d81470 100644--- a/apps/server/src/provider/Layers/EventNdjsonLogger.ts+++ b/apps/server/src/provider/Layers/EventNdjsonLogger.ts@@ -5,7 +5,8 @@* Native and canonical views share batching, rotation, and retention state so* they cannot race while appending to the same thread-scoped file.*/-import * as NodeFS from "node:fs";+import type * as NodeFS from "node:fs";+import * as NodeFSP from "node:fs/promises";import * as NodePath from "node:path";import type { ThreadId } from "@t3tools/contracts";@@ -57,6 +58,7 @@ export interface EventNdjsonLogger {export interface EventNdjsonLogStore {readonly filePath: string;readonly logger: (stream: EventNdjsonStream) => EventNdjsonLogger;+ readonly flush: Effect.Effect<void>;readonly close: () => Effect.Effect<void>;}@@ -189,19 +191,19 @@ function shouldPersist(stream: EventNdjsonStream, event: unknown): boolean {}}-export function writeBatchedMessages(+export async function writeBatchedMessages(sink: Pick<RotatingFileSink, "write">,records: ReadonlyArray<PendingRecord>,maxBytes: number,onWritten: (records: ReadonlyArray<PendingRecord>) => void,-): void {+): Promise<void> {let pendingRecords: Array<PendingRecord> = [];let pendingBytes = 0;- const flush = () => {+ const flush = async () => {if (pendingRecords.length === 0) return;const writtenRecords = pendingRecords;- sink.write(writtenRecords.map((record) => record.line).join(""));+ await sink.write(writtenRecords.map((record) => record.line).join(""));onWritten(writtenRecords);pendingRecords = [];pendingBytes = 0;@@ -209,45 +211,49 @@ export function writeBatchedMessages(for (const record of records) {if (pendingBytes > 0 && pendingBytes + record.bytes > maxBytes) {- flush();+ await flush();}pendingRecords.push(record);pendingBytes += record.bytes;if (pendingBytes >= maxBytes) {- flush();+ await flush();}}- flush();+ await flush();}-function isProviderLogFile(filePath: string, fileName: string, filePrefix: string): boolean {+async function isProviderLogFile(+ filePath: string,+ fileName: string,+ filePrefix: string,+): Promise<boolean> {if (!/\.log(?:\.\d+)?$/u.test(fileName)) return false;if (fileName.startsWith(filePrefix)) return true;- const descriptor = NodeFS.openSync(filePath, "r");+ const descriptor = await NodeFSP.open(filePath, "r");try {const header = Buffer.alloc(256);- const bytesRead = NodeFS.readSync(descriptor, header, 0, header.byteLength, 0);+ const { bytesRead } = await descriptor.read(header, 0, header.byteLength, 0);return /^\[[^\]\r\n]+\] (?:NTIVE|CANON|ORCH): /u.test(header.toString("utf8", 0, bytesRead));} finally {- NodeFS.closeSync(descriptor);+ await descriptor.close();}}-function enforceRetention(input: {+async function enforceRetention(input: {readonly directory: string;readonly maxTotalBytes: number;readonly maxAgeMs: number;readonly activeFilePaths: ReadonlySet<string>;readonly filePrefix: string;readonly now: number;-}): RetentionResult {+}): Promise<RetentionResult> {const failures: Array<FileOperationFailure> = [];const files: Array<{ filePath: string; mtimeMs: number; size: number }> = [];let entries: ReadonlyArray<NodeFS.Dirent>;try {- entries = NodeFS.readdirSync(input.directory, { withFileTypes: true });+ entries = await NodeFSP.readdir(input.directory, { withFileTypes: true });} catch (cause) {return { failures: [{ filePath: input.directory, cause }] };}@@ -256,8 +262,8 @@ function enforceRetention(input: {if (!entry.isFile()) continue;const filePath = NodePath.join(input.directory, entry.name);try {- if (!isProviderLogFile(filePath, entry.name, input.filePrefix)) continue;- const stat = NodeFS.statSync(filePath);+ if (!(await isProviderLogFile(filePath, entry.name, input.filePrefix))) continue;+ const stat = await NodeFSP.stat(filePath);files.push({ filePath, mtimeMs: stat.mtimeMs, size: stat.size });} catch (cause) {failures.push({ filePath, cause });@@ -265,10 +271,10 @@ function enforceRetention(input: {}let totalBytes = files.reduce((total, file) => total + file.size, 0);- const remove = (file: (typeof files)[number]) => {+ const remove = async (file: (typeof files)[number]) => {if (input.activeFilePaths.has(file.filePath)) return false;try {- NodeFS.rmSync(file.filePath, { force: true });+ await NodeFSP.rm(file.filePath, { force: true });totalBytes -= file.size;return true;} catch (cause) {@@ -277,16 +283,16 @@ function enforceRetention(input: {}};- const retained = files.filter((file) => {- if (input.now - file.mtimeMs <= input.maxAgeMs) return true;- return !remove(file);- });+ const retained: typeof files = [];+ for (const file of files) {+ if (input.now - file.mtimeMs <= input.maxAgeMs || !(await remove(file))) retained.push(file);+ }for (const file of retained.toSorted((left, right) => left.mtimeMs - right.mtimeMs || left.filePath.localeCompare(right.filePath),)) {if (totalBytes <= input.maxTotalBytes) break;- remove(file);+ await remove(file);}return { failures };@@ -337,7 +343,7 @@ function resolveOptions(return Effect.succeed(resolved);}-function drainPending(input: {+async function drainPending(input: {readonly directory: string;readonly options: ResolvedOptions;readonly state: StoreState;@@ -345,7 +351,7 @@ function drainPending(input: {readonly now: number;readonly timerFired: boolean;readonly close: boolean;-}): readonly [DrainResult, StoreState] {+}): Promise<readonly [DrainResult, StoreState]> {if (input.state.closed) {return [{ attributions: [], failures: [] }, input.state];}@@ -373,7 +379,7 @@ function drainPending(input: {filePath,maxBytes: input.options.maxBytes,maxFiles: input.options.maxFiles,- throwOnError: true,+ maxBufferedBytes: input.options.maxBufferedBytes,});sinks.set(threadSegment, sink);} catch (cause) {@@ -383,7 +389,7 @@ function drainPending(input: {}try {- writeBatchedMessages(sink, records, input.options.maxBytes, (writtenRecords) => {+ await writeBatchedMessages(sink, records, input.options.maxBytes, (writtenRecords) => {for (const record of writtenRecords) {const current = attributionByStream.get(record.stream) ?? {count: 0,@@ -396,6 +402,7 @@ function drainPending(input: {}});} catch (cause) {+ await sink.close().catch(() => undefined);sinks.delete(threadSegment);failures.push({ filePath, cause });}@@ -404,7 +411,7 @@ function drainPending(input: {const retentionDue =input.now - input.state.lastRetentionAt >= input.options.retentionCheckIntervalMs;const retention = retentionDue- ? enforceRetention({+ ? await enforceRetention({directory: input.directory,maxTotalBytes: input.options.maxTotalBytes,maxAgeMs: input.options.maxAgeMs,@@ -418,6 +425,20 @@ function drainPending(input: {}): { failures: [] };+ if (input.close) {+ for (const [segment, sink] of sinks) {+ try {+ await sink.close();+ } catch (cause) {+ failures.push({+ filePath: providerLogPath(input.directory, input.filePrefix, segment),+ cause,+ });+ }+ }+ sinks.clear();+ }+return [{attributions: Array.from(attributionByStream, ([stream, value]) => ({@@ -455,13 +476,13 @@ export const makeEventNdjsonLogStore = Effect.fnUntraced(function* (const directory = NodePath.dirname(filePath);const filePrefix = providerLogPrefix(filePath);- yield* Effect.try({- try: () => NodeFS.mkdirSync(directory, { recursive: true }),+ yield* Effect.tryPromise({+ try: () => NodeFSP.mkdir(directory, { recursive: true }),catch: (cause) => new EventNdjsonLogDirectoryError({ directory, cause }),});const initializedAt = yield* Clock.currentTimeMillis;- const initialRetention = yield* Effect.sync(() =>+ const initialRetention = yield* Effect.promise(() =>enforceRetention({directory,maxTotalBytes: resolved.maxTotalBytes,@@ -488,22 +509,12 @@ export const makeEventNdjsonLogStore = Effect.fnUntraced(function* (});const timerScope = yield* Scope.make();- const flush = Effect.fnUntraced(function* (timerFired: boolean, close: boolean) {- const startedAt = yield* Clock.currentTimeMillis;- const result = yield* SynchronizedRef.modifyEffect(stateRef, (state) =>- Effect.sync(() =>- drainPending({- directory,- options: resolved,- state,- filePrefix,- now: startedAt,- timerFired,- close,- }),- ),+ const drain = (state: StoreState, now: number, timerFired = false, close = false) =>+ Effect.promise(() =>+ drainPending({ directory, options: resolved, state, filePrefix, now, timerFired, close }),);+ const reportDrain = Effect.fnUntraced(function* (result: DrainResult, startedAt: number) {for (const failure of result.failures) {yield* logWarning("provider event log write or retention failed", {filePath: failure.filePath,@@ -536,6 +547,14 @@ export const makeEventNdjsonLogStore = Effect.fnUntraced(function* (}});+ const flush = Effect.fnUntraced(function* (timerFired: boolean, close: boolean) {+ const startedAt = yield* Clock.currentTimeMillis;+ const result = yield* SynchronizedRef.modifyEffect(stateRef, (state) =>+ drain(state, startedAt, timerFired, close),+ );+ yield* reportDrain(result, startedAt);+ }, Effect.uninterruptible);+const scheduleFlush = Effect.fnUntraced(function* () {yield* Effect.forkIn(Effect.sleep(resolved.batchWindowMs).pipe(Effect.andThen(flush(true, false))),@@ -547,7 +566,7 @@ export const makeEventNdjsonLogStore = Effect.fnUntraced(function* (const close = Effect.fnUntraced(function* () {yield* flush(false, true);yield* Scope.close(timerScope, Exit.void);- });+ }, Effect.uninterruptible);const loggerViews = new Map<EventNdjsonStream, EventNdjsonLogger>();const logger = (stream: EventNdjsonStream): EventNdjsonLogger => {@@ -556,40 +575,51 @@ export const makeEventNdjsonLogStore = Effect.fnUntraced(function* (const write = Effect.fnUntraced(function* (event: unknown, threadId: ThreadId | null) {if (!shouldPersist(stream, event)) return;- const payload = yield* serializeEvent(event);- if (payload === undefined) return;-- const observedAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));- const line = `[${observedAt}] ${resolveStreamLabel(stream)}: ${payload}\n`;- const bytes = Buffer.byteLength(line);- const action = yield* SynchronizedRef.modifyEffect(stateRef, (state) => {- if (state.closed) {- return Effect.succeed([{ flush: false }, state] as const);- }- const pending = [- ...state.pending,- { stream, threadSegment: resolveThreadSegment(threadId), line, bytes },- ];- const pendingBytes = state.pendingBytes + bytes;- const flush =- resolved.batchWindowMs === 0 ||- pending.length >= resolved.maxBufferedRecords ||- pendingBytes >= resolved.maxBufferedBytes;- const schedule = !flush && !state.flushScheduled;- const nextState = {- ...state,- pending,- pendingBytes,- flushScheduled: state.flushScheduled || schedule,- };- return (schedule ? scheduleFlush() : Effect.void).pipe(- Effect.as([{ flush }, nextState] as const),- );- }).pipe(Effect.uninterruptible);-- if (action.flush) {- yield* flush(false, false);- }+ const startedAt = yield* Clock.currentTimeMillis;+ const results = yield* SynchronizedRef.modifyEffect(stateRef, (state) =>+ Effect.gen(function* () {+ const results: DrainResult[] = [];+ if (state.closed) return [results, state] as const;+ const payload = yield* serializeEvent(event);+ if (payload === undefined) return [results, state] as const;+ const observedAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));+ const line = `[${observedAt}] ${resolveStreamLabel(stream)}: ${payload}\n`;+ const bytes = Buffer.byteLength(line);+ if (bytes > resolved.maxBufferedBytes) {+ yield* logWarning("provider event log record exceeds buffer capacity", {+ filePath,+ bytes,+ });+ return [results, state] as const;+ }+ let current = state;+ if (current.pendingBytes + bytes > resolved.maxBufferedBytes) {+ const [result, drained] = yield* drain(current, startedAt);+ results.push(result);+ current = drained;+ }+ const pending = [+ ...current.pending,+ { stream, threadSegment: resolveThreadSegment(threadId), line, bytes },+ ];+ const pendingBytes = current.pendingBytes + bytes;+ const flushNow =+ resolved.batchWindowMs === 0 ||+ pending.length >= resolved.maxBufferedRecords ||+ pendingBytes >= resolved.maxBufferedBytes;+ current = { ...current, pending, pendingBytes };+ if (flushNow) {+ const [result, drained] = yield* drain(current, startedAt);+ results.push(result);+ current = drained;+ } else if (!current.flushScheduled) {+ yield* scheduleFlush();+ current = { ...current, flushScheduled: true };+ }+ return [results, current] as const;+ }),+ ).pipe(Effect.uninterruptible);+ for (const result of results) yield* reportDrain(result, startedAt);});const view = { filePath, write, close: () => Effect.void } satisfies EventNdjsonLogger;@@ -597,7 +627,7 @@ export const makeEventNdjsonLogStore = Effect.fnUntraced(function* (return view;};- return { filePath, logger, close } satisfies EventNdjsonLogStore;+ return { filePath, logger, flush: flush(false, false), close } satisfies EventNdjsonLogStore;});export const makeEventNdjsonLogger = Effect.fnUntraced(function* (diff --git a/apps/server/src/resourceTelemetry/DesktopTelemetryReceiver.test.ts b/apps/server/src/resourceTelemetry/DesktopTelemetryReceiver.test.tsindex 50c8ea2b7..5fbafb74a 100644--- a/apps/server/src/resourceTelemetry/DesktopTelemetryReceiver.test.ts+++ b/apps/server/src/resourceTelemetry/DesktopTelemetryReceiver.test.ts@@ -1,9 +1,13 @@// @effect-diagnostics nodeBuiltinImport:off+import * as NodeChildProcess from "node:child_process";import * as NodeFS from "node:fs";import * as NodeOS from "node:os";import * as NodePath from "node:path";+import * as NodeStream from "node:stream";import { it } from "@effect/vitest";+import { HostProcessPlatform } from "@t3tools/shared/hostProcess";+import * as Context from "effect/Context";import * as DateTime from "effect/DateTime";import * as Effect from "effect/Effect";import * as Option from "effect/Option";@@ -15,6 +19,7 @@ import {type DesktopTelemetryReceiverHealth,initialDesktopTelemetryContactAt,isDesktopTelemetryContactStale,+ openDesktopTelemetryReadable,recordDesktopTelemetrySampleHealth,requireDesktopTelemetryWriteProgress,resolveDesktopTelemetrySnapshotStaleAfterMs,@@ -22,6 +27,122 @@ import {} from "./DesktopTelemetryReceiver.ts";describe("DesktopTelemetryReceiver", () => {+ it("reads and closes an owned regular-file descriptor", async () => {+ const directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-telemetry-file-"));+ const path = NodePath.join(directory, "telemetry.ndjson");+ const payload = '{"type":"desktopTelemetryHello","version":1}\n';+ NodeFS.writeFileSync(path, payload);+ const fd = NodeFS.openSync(path, "r");+ const readable = openDesktopTelemetryReadable(fd);+ try {+ expect(readable).toBeInstanceOf(NodeFS.ReadStream);+ const closed = new Promise<void>((resolve, reject) => {+ readable.once("close", resolve);+ readable.once("error", reject);+ });+ let received = "";+ readable.setEncoding("utf8");+ for await (const chunk of readable) received += chunk;+ await closed;+ expect(received).toBe(payload);+ expect(() => NodeFS.fstatSync(fd)).toThrow();+ } finally {+ readable.destroy();+ NodeFS.rmSync(directory, { recursive: true, force: true });+ }+ });++ it.skipIf(Context.get(Context.empty(), HostProcessPlatform) === "win32")(+ "closes an inherited socket and exits with its writer still open without filesystem reads",+ async ({ onTestFinished }) => {+ const sourceUrl = new URL("./DesktopTelemetryReceiver.ts", import.meta.url).href;+ const child = NodeChildProcess.spawn(+ process.execPath,+ [+ "--input-type=module",+ "-e",+ `+ import assert from "node:assert/strict";+ import fs from "node:fs";+ import * as NodeNet from "node:net";+ import * as NodeEvents from "node:events";+ import { syncBuiltinESMExports } from "node:module";+ import { openDesktopTelemetryReadable } from ${JSON.stringify(sourceUrl)};++ assert.equal(fs.fstatSync(3).isSocket(), true);+ const originalRead = fs.read;+ fs.read = (fd, ...args) => {+ assert.notEqual(fd, 3, "telemetry must not queue a blocking filesystem read");+ return originalRead(fd, ...args);+ };+ syncBuiltinESMExports();+ const readable = openDesktopTelemetryReadable(3);+ if (!(readable instanceof NodeNet.Socket)) {+ process.send("filesystem-reader");+ readable.destroy();+ process.disconnect();+ } else {+ let received = "";+ readable.on("data", (chunk) => {+ received += chunk.toString();+ if (!received.endsWith("\\n")) return;+ assert.equal(received, "telemetry\\n");+ process.send("read");+ });+ process.once("message", async (message) => {+ assert.equal(message, "close");+ const closed = NodeEvents.once(readable, "close");+ readable.destroy();+ await closed;+ await fs.promises.stat(new URL(${JSON.stringify(sourceUrl)}));+ process.send("closed");+ process.disconnect();+ });+ }+ `,+ ],+ {+ env: { ...process.env, UV_THREADPOOL_SIZE: "1" },+ stdio: ["ignore", "ignore", "pipe", "pipe", "ipc"],+ },+ );+ const writer = child.stdio[3];+ assert.instanceOf(writer, NodeStream.Duplex);+ if (!(writer instanceof NodeStream.Duplex)) throw new Error("Missing telemetry writer");+ writer.allowHalfOpen = true;+ onTestFinished(() => {+ writer.destroy();+ if (child.exitCode === null && child.signalCode === null) child.kill();+ });+ let stderr = "";+ child.stderr?.setEncoding("utf8").on("data", (chunk: string) => {+ stderr += chunk;+ });+ const exited = new Promise<readonly [number | null, NodeJS.Signals | null]>(+ (resolve, reject) => {+ child.once("exit", (code, signal) => resolve([code, signal]));+ child.once("error", reject);+ },+ );+ const nextMessage = () =>+ Promise.race([+ new Promise<unknown>((resolve) => child.once("message", resolve)),+ exited.then(([code, signal]) => {+ throw new Error(`Telemetry fixture exited early (${code}, ${signal}): ${stderr}`);+ }),+ ]);+ const read = nextMessage();+ writer.write("telemetry\n");+ expect(await read).toBe("read");+ expect(writer.writableEnded).toBe(false);+ const closed = nextMessage();+ child.send("close");+ expect(await closed).toBe("closed");+ expect(await exited).toEqual([0, null]);+ expect(writer.writableEnded).toBe(false);+ },+ );+it("degrades a hello-only stream after the first-sample deadline", () => {expect(isDesktopTelemetryContactStale(Option.some(1_000), 90_999)).toBe(false);expect(isDesktopTelemetryContactStale(Option.some(1_000), 91_000)).toBe(true);diff --git a/apps/server/src/resourceTelemetry/DesktopTelemetryReceiver.ts b/apps/server/src/resourceTelemetry/DesktopTelemetryReceiver.tsindex f3fb4a525..69b704ed1 100644--- a/apps/server/src/resourceTelemetry/DesktopTelemetryReceiver.ts+++ b/apps/server/src/resourceTelemetry/DesktopTelemetryReceiver.ts@@ -1,5 +1,6 @@// @effect-diagnostics nodeBuiltinImport:offimport * as NodeFS from "node:fs";+import * as NodeNet from "node:net";import * as NodeStream from "@effect/platform-node/NodeStream";import {@@ -303,6 +304,15 @@ export function requireDesktopTelemetryWriteProgress(: Effect.fail(new DesktopTelemetryControlStalled({ fd, remainingBytes }));}+export function openDesktopTelemetryReadable(fd: number) {+ // Filesystem reads on inherited sockets cannot be cancelled while the peer stays open.+ if (NodeFS.fstatSync(fd).isSocket()) {+ return new NodeNet.Socket({ fd, readable: true, writable: false });+ }+ // Keep file, FIFO, and Windows non-socket descriptor handling unchanged.+ return NodeFS.createReadStream("", { fd, autoClose: true });+}+export const make = Effect.fn("resourceTelemetry.desktopTelemetryReceiver.make")(function* () {const config = yield* ServerConfig;const serverSettings = yield* ServerSettingsService;@@ -438,11 +448,7 @@ export const make = Effect.fn("resourceTelemetry.desktopTelemetryReceiver.make")const fd = config.desktopTelemetryFd;const readable = yield* Effect.acquireRelease(Effect.try({- try: () =>- NodeFS.createReadStream("", {- fd,- autoClose: true,- }),+ try: () => openDesktopTelemetryReadable(fd),catch: (cause) => new DesktopTelemetryStreamFailed({ fd, cause }),}),(stream) =>diff --git a/packages/shared/src/logging.test.ts b/packages/shared/src/logging.test.tsindex 4b19bd291..efffb5053 100644--- a/packages/shared/src/logging.test.ts+++ b/packages/shared/src/logging.test.ts@@ -4,167 +4,145 @@ import * as NodeOS from "node:os";import * as NodePath from "node:path";import { afterEach, describe, expect, it } from "vite-plus/test";-import {- RotatingFileSink,- RotatingFileSinkConfigurationError,- RotatingFileSinkError,-} from "./logging.ts";+import { RotatingFileSink, RotatingFileSinkConfigurationError } from "./logging.ts";const tempDirectories: string[] = [];-const makeTempDirectory = (): string => {const directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3code-logging-"));tempDirectories.push(directory);return directory;};-const captureError = (run: () => unknown): unknown => {- try {- run();- } catch (cause) {- return cause;- }- throw new Error("Expected operation to throw");-};-afterEach(() => {- for (const directory of tempDirectories.splice(0)) {+ for (const directory of tempDirectories.splice(0))NodeFS.rmSync(directory, { recursive: true, force: true });- }});describe("RotatingFileSink", () => {- it.each([- { option: "maxBytes" as const, maxBytes: 0, maxFiles: 1 },- { option: "maxFiles" as const, maxBytes: 1, maxFiles: 0 },- ])("reports invalid $option configuration structurally", (input) => {- const thrown = captureError(- () =>- new RotatingFileSink({- filePath: "/unused/log.ndjson",- maxBytes: input.maxBytes,- maxFiles: input.maxFiles,- }),- );-- expect(thrown).toBeInstanceOf(RotatingFileSinkConfigurationError);- expect(thrown).toMatchObject({- option: input.option,- received: 0,- minimum: 1,- });- expect((thrown as Error).message).toBe(`${input.option} must be >= 1 (received 0)`);- });-- it("preserves directory initialization failures", () => {- const directory = makeTempDirectory();- const parentFile = NodePath.join(directory, "not-a-directory");- const filePath = NodePath.join(parentFile, "log.ndjson");+ it.each(["maxBytes", "maxFiles", "maxBufferedBytes", "maxBufferedChunks"] as const)(+ "validates %s before starting I/O",+ (option) => {+ expect(+ () =>+ new RotatingFileSink({ filePath: "/unused/log", maxBytes: 1, maxFiles: 1, [option]: 0 }),+ ).toThrow(RotatingFileSinkConfigurationError);+ },+ );++ it("reports asynchronous initialization failures on flush and close", async () => {+ const parentFile = NodePath.join(makeTempDirectory(), "not-a-directory");NodeFS.writeFileSync(parentFile, "occupied");-- const thrown = captureError(() => new RotatingFileSink({ filePath, maxBytes: 1, maxFiles: 1 }));-- expect(thrown).toBeInstanceOf(RotatingFileSinkError);- expect(thrown).toMatchObject({ operation: "initialize", filePath });- expect((thrown as RotatingFileSinkError).cause).toBeInstanceOf(Error);+ const filePath = NodePath.join(parentFile, "log");+ const sink = new RotatingFileSink({ filePath, maxBytes: 10, maxFiles: 1 });+ await expect(sink.flush()).rejects.toMatchObject({ operation: "initialize", filePath });+ await expect(sink.close()).rejects.toMatchObject({ operation: "initialize", filePath });});- it("only treats a missing log file as an empty current size", () => {- const directory = makeTempDirectory();- const filePath = NodePath.join(directory, "a".repeat(300));-- const thrown = captureError(() => new RotatingFileSink({ filePath, maxBytes: 1, maxFiles: 1 }));-- expect(thrown).toBeInstanceOf(RotatingFileSinkError);- expect(thrown).toMatchObject({ operation: "read", filePath });- expect((thrown as RotatingFileSinkError).cause).toMatchObject({ code: "ENAMETOOLONG" });+ it("only treats a missing file as zero bytes", async () => {+ const filePath = NodePath.join(makeTempDirectory(), "a".repeat(300));+ const sink = new RotatingFileSink({ filePath, maxBytes: 10, maxFiles: 1 });+ await expect(sink.close()).rejects.toMatchObject({+ operation: "read",+ cause: { code: "ENAMETOOLONG" },+ });});- it("starts an absent log file at zero bytes", () => {- const directory = makeTempDirectory();- const filePath = NodePath.join(directory, "log.ndjson");- const sink = new RotatingFileSink({ filePath, maxBytes: 100, maxFiles: 1 });-- sink.write("entry");+ it("orders concurrent appends across rotation and drains on close", async () => {+ const filePath = NodePath.join(makeTempDirectory(), "log");+ const sink = new RotatingFileSink({ filePath, maxBytes: 10, maxFiles: 5 });+ const lines = Array.from({ length: 20 }, (_, index) => `${String(index).padStart(2, "0")}\n`);+ const writes = lines.map((line) => sink.write(line));+ await sink.close();+ await Promise.all(writes);+ const retained = [5, 4, 3, 2, 1, 0]+ .map((index) => NodeFS.readFileSync(index === 0 ? filePath : `${filePath}.${index}`, "utf8"))+ .join("");+ expect(retained).toBe(lines.slice(3).join(""));+ expect(NodeFS.existsSync(`${filePath}.6`)).toBe(false);+ expect(sink.bufferedBytes).toBe(0);+ expect(sink.bufferedChunks).toBe(0);+ await expect(sink.write("late")).rejects.toMatchObject({ operation: "closed" });+ await sink.close();+ });- expect(NodeFS.readFileSync(filePath, "utf8")).toBe("entry");+ it("counts UTF-8 bytes and owns buffers until flush", async () => {+ const filePath = NodePath.join(makeTempDirectory(), "log");+ const sink = new RotatingFileSink({ filePath, maxBytes: 6, maxFiles: 1 });+ const buffer = Buffer.from("éé");+ const first = sink.write(buffer);+ buffer.fill(0);+ const second = sink.write("🙂");+ await sink.flush();+ await Promise.all([first, second]);+ expect(NodeFS.readFileSync(`${filePath}.1`, "utf8")).toBe("éé");+ expect(NodeFS.readFileSync(filePath, "utf8")).toBe("🙂");+ await sink.close();});- it("preserves write failures", () => {- const directory = makeTempDirectory();- const filePath = NodePath.join(directory, "log.ndjson");- NodeFS.mkdirSync(filePath);+ it("bounds accepted bytes without dropping or retrying accepted chunks", async () => {+ const filePath = NodePath.join(makeTempDirectory(), "log");const sink = new RotatingFileSink({filePath,- maxBytes: Number.MAX_SAFE_INTEGER,+ maxBytes: 100,maxFiles: 1,- throwOnError: true,+ maxBufferedBytes: 4,});-- const thrown = captureError(() => sink.write("entry"));-- expect(thrown).toBeInstanceOf(RotatingFileSinkError);- expect(thrown).toMatchObject({ operation: "write", filePath });- expect((thrown as RotatingFileSinkError).cause).toMatchObject({ code: "EISDIR" });+ const first = sink.write("🙂");+ expect(sink.bufferedBytes).toBe(4);+ await expect(sink.write("x")).rejects.toMatchObject({ operation: "buffer" });+ await first;+ await sink.write("ok");+ await expect(sink.close()).rejects.toMatchObject({ operation: "buffer" });+ expect(NodeFS.readFileSync(filePath, "utf8")).toBe("🙂ok");});- it("preserves rotation failures without an artificial write wrapper", () => {- const directory = makeTempDirectory();- const filePath = NodePath.join(directory, "log.ndjson");- NodeFS.writeFileSync(filePath, "a");- NodeFS.mkdirSync(`${filePath}.1`);+ it("bounds the number of queued chunks independently", async () => {+ const filePath = NodePath.join(makeTempDirectory(), "log");const sink = new RotatingFileSink({filePath,- maxBytes: 1,+ maxBytes: 100,maxFiles: 1,- throwOnError: true,+ maxBufferedChunks: 1,});-- const thrown = captureError(() => sink.write("b"));-- expect(thrown).toBeInstanceOf(RotatingFileSinkError);- expect(thrown).toMatchObject({ operation: "rotate", filePath });- expect((thrown as RotatingFileSinkError).cause).toBeInstanceOf(Error);+ const first = sink.write("a");+ await expect(sink.write("b")).rejects.toMatchObject({ operation: "buffer" });+ await first;+ await expect(sink.close()).rejects.toMatchObject({ operation: "buffer" });});- it("never reports a rotation failure after successfully appending a chunk", () => {- const directory = makeTempDirectory();- const filePath = NodePath.join(directory, "log.ndjson");- NodeFS.mkdirSync(`${filePath}.1`);- const sink = new RotatingFileSink({- filePath,- maxBytes: 1,- maxFiles: 1,- throwOnError: true,+ it("fails queued writes and final flush after a write failure", async () => {+ const filePath = NodePath.join(makeTempDirectory(), "log");+ NodeFS.mkdirSync(filePath);+ const sink = new RotatingFileSink({ filePath, maxBytes: Number.MAX_SAFE_INTEGER, maxFiles: 1 });+ const writes = [sink.write("a"), sink.write("b")];+ const results = await Promise.allSettled(writes);+ expect(results.map((result) => result.status)).toEqual(["rejected", "rejected"]);+ await expect(sink.flush()).rejects.toMatchObject({+ operation: "write",+ cause: { code: "EISDIR" },});+ await expect(sink.close()).rejects.toMatchObject({ operation: "write" });+ expect(sink.bufferedBytes).toBe(0);+ });- sink.write("oversized");-- expect(NodeFS.readFileSync(filePath, "utf8")).toBe("oversized");- const thrown = captureError(() => sink.write("next"));- expect(thrown).toMatchObject({ operation: "rotate", filePath });+ it("reports rotation errors before appending, including after an oversized record", async () => {+ const filePath = NodePath.join(makeTempDirectory(), "log");+ NodeFS.mkdirSync(`${filePath}.1`);+ const sink = new RotatingFileSink({ filePath, maxBytes: 1, maxFiles: 1 });+ await sink.write("oversized");+ await expect(sink.write("next")).rejects.toMatchObject({ operation: "rotate" });+ await expect(sink.close()).rejects.toMatchObject({ operation: "rotate" });expect(NodeFS.readFileSync(filePath, "utf8")).toBe("oversized");});- it("preserves backup pruning failures", () => {- const directory = makeTempDirectory();- const filePath = NodePath.join(directory, "log.ndjson");- const overflowBackup = `${filePath}.2`;- NodeFS.mkdirSync(overflowBackup);- NodeFS.writeFileSync(NodePath.join(overflowBackup, "entry"), "occupied");-- const thrown = captureError(- () =>- new RotatingFileSink({- filePath,- maxBytes: 1,- maxFiles: 1,- throwOnError: true,- }),- );-- expect(thrown).toBeInstanceOf(RotatingFileSinkError);- expect(thrown).toMatchObject({ operation: "prune", filePath });- expect((thrown as RotatingFileSinkError).cause).toBeInstanceOf(Error);+ it("prunes overflow backups asynchronously and reports pruning failures", async () => {+ const filePath = NodePath.join(makeTempDirectory(), "log");+ NodeFS.writeFileSync(`${filePath}.3`, "old");+ const sink = new RotatingFileSink({ filePath, maxBytes: 10, maxFiles: 1 });+ await sink.close();+ expect(NodeFS.existsSync(`${filePath}.3`)).toBe(false);+ NodeFS.mkdirSync(`${filePath}.2`);+ const broken = new RotatingFileSink({ filePath, maxBytes: 10, maxFiles: 1 });+ await expect(broken.close()).rejects.toMatchObject({ operation: "prune" });});});diff --git a/packages/shared/src/logging.ts b/packages/shared/src/logging.tsindex ac3d56e28..2acc04bdd 100644--- a/packages/shared/src/logging.ts+++ b/packages/shared/src/logging.ts@@ -1,5 +1,5 @@// @effect-diagnostics nodeBuiltinImport:off-import * as NodeFS from "node:fs";+import * as NodeFSP from "node:fs/promises";import * as NodePath from "node:path";import * as Schema from "effect/Schema";@@ -7,13 +7,14 @@ export interface RotatingFileSinkOptions {readonly filePath: string;readonly maxBytes: number;readonly maxFiles: number;- readonly throwOnError?: boolean;+ readonly maxBufferedBytes?: number;+ readonly maxBufferedChunks?: number;}export class RotatingFileSinkConfigurationError extends Schema.TaggedErrorClass<RotatingFileSinkConfigurationError>()("RotatingFileSinkConfigurationError",{- option: Schema.Literals(["maxBytes", "maxFiles"]),+ option: Schema.Literals(["maxBytes", "maxFiles", "maxBufferedBytes", "maxBufferedChunks"]),received: Schema.Number,minimum: Schema.Number,},@@ -26,7 +27,15 @@ export class RotatingFileSinkConfigurationError extends Schema.TaggedErrorClass<export class RotatingFileSinkError extends Schema.TaggedErrorClass<RotatingFileSinkError>()("RotatingFileSinkError",{- operation: Schema.Literals(["initialize", "read", "write", "rotate", "prune"]),+ operation: Schema.Literals([+ "initialize",+ "read",+ "write",+ "rotate",+ "prune",+ "buffer",+ "closed",+ ]),filePath: Schema.String,cause: Schema.Defect(),},@@ -37,146 +46,208 @@ export class RotatingFileSinkError extends Schema.TaggedErrorClass<RotatingFileS}const isRotatingFileSinkError = Schema.is(RotatingFileSinkError);-const isFileNotFoundError = (cause: unknown): cause is NodeJS.ErrnoException =>cause instanceof Error && "code" in cause && cause.code === "ENOENT";+interface PendingChunk {+ readonly buffer: Buffer;+ readonly resolve: () => void;+ readonly reject: (error: RotatingFileSinkError) => void;+}++const MAX_APPEND_BATCH_BYTES = 64 * 1024;++/** One owner serializes initialization, appends and rotation. Flush acknowledges accepted writes, not fsync. */export class RotatingFileSink {private readonly filePath: string;private readonly maxBytes: number;private readonly maxFiles: number;- private readonly throwOnError: boolean;+ private readonly maxBufferedBytes: number;+ private readonly maxBufferedChunks: number;private currentSize = 0;+ private pendingBytes = 0;+ private pendingChunks = 0;+ private closed = false;+ private failure: RotatingFileSinkError | undefined;+ private fatalFailure: RotatingFileSinkError | undefined;+ private readonly initialized: Promise<void>;+ private latestWrite: Promise<void> | undefined;+ private readonly queue: PendingChunk[] = [];+ private draining = false;constructor(options: RotatingFileSinkOptions) {- if (options.maxBytes < 1) {- throw new RotatingFileSinkConfigurationError({- option: "maxBytes",- received: options.maxBytes,- minimum: 1,- });- }- if (options.maxFiles < 1) {- throw new RotatingFileSinkConfigurationError({- option: "maxFiles",- received: options.maxFiles,- minimum: 1,- });+ const limits = {+ maxBytes: options.maxBytes,+ maxFiles: options.maxFiles,+ maxBufferedBytes: options.maxBufferedBytes ?? 1024 * 1024,+ maxBufferedChunks: options.maxBufferedChunks ?? 512,+ };+ for (const [option, received] of Object.entries(limits)) {+ if (!Number.isSafeInteger(received) || received < 1) {+ throw new RotatingFileSinkConfigurationError({+ option: option as keyof typeof limits,+ received,+ minimum: 1,+ });+ }}-this.filePath = options.filePath;- this.maxBytes = options.maxBytes;- this.maxFiles = options.maxFiles;- this.throwOnError = options.throwOnError ?? false;+ this.maxBytes = limits.maxBytes;+ this.maxFiles = limits.maxFiles;+ this.maxBufferedBytes = limits.maxBufferedBytes;+ this.maxBufferedChunks = limits.maxBufferedChunks;+ this.initialized = this.initialize().catch((cause: RotatingFileSinkError) => {+ this.failure = cause;+ this.fatalFailure = cause;+ });+ }- try {- NodeFS.mkdirSync(NodePath.dirname(this.filePath), { recursive: true });- } catch (cause) {- throw new RotatingFileSinkError({- operation: "initialize",- filePath: this.filePath,- cause,- });- }- this.pruneOverflowBackups();- this.currentSize = this.readCurrentSize();+ get bufferedBytes(): number {+ return this.pendingBytes;}- write(chunk: string | Buffer): void {- const buffer = typeof chunk === "string" ? Buffer.from(chunk) : chunk;- if (buffer.length === 0) return;+ get bufferedChunks(): number {+ return this.pendingChunks;+ }- try {- if (this.currentSize > 0 && this.currentSize + buffer.length > this.maxBytes) {- this.rotate();- }+ write(chunk: string | Buffer): Promise<void> {+ const bytes = typeof chunk === "string" ? Buffer.byteLength(chunk) : chunk.length;+ if (this.closed) {+ return Promise.reject(this.error("closed", new Error("Log sink is closed")));+ }+ if (bytes === 0) return Promise.resolve();+ if (+ this.pendingBytes + bytes > this.maxBufferedBytes ||+ this.pendingChunks >= this.maxBufferedChunks+ ) {+ const error = this.error("buffer", new Error("Log buffer capacity exceeded"));+ this.failure ??= error;+ return Promise.reject(error);+ }- NodeFS.appendFileSync(this.filePath, buffer);- this.currentSize += buffer.length;- } catch (cause) {- if (isRotatingFileSinkError(cause)) {- throw cause;- }- if (this.throwOnError) {- throw new RotatingFileSinkError({- operation: "write",- filePath: this.filePath,- cause,- });- }- this.currentSize = this.readCurrentSize();+ // Copy caller-owned buffers so queued bytes cannot change before the append.+ const buffer = Buffer.from(chunk);+ this.pendingBytes += bytes;+ this.pendingChunks += 1;+ const receipt = Promise.withResolvers<void>();+ this.queue.push({ buffer, resolve: () => receipt.resolve(), reject: receipt.reject });+ this.latestWrite = receipt.promise;+ void receipt.promise.catch(() => undefined);+ if (!this.draining) {+ this.draining = true;+ void this.drain();}+ return receipt.promise;}- private rotate(): void {- try {- const oldest = this.withSuffix(this.maxFiles);- if (NodeFS.existsSync(oldest)) {- NodeFS.rmSync(oldest, { force: true });- }+ async flush(): Promise<void> {+ await (this.latestWrite ?? this.initialized).catch(() => undefined);+ if (this.failure) throw this.failure;+ }- for (let index = this.maxFiles - 1; index >= 1; index -= 1) {- const source = this.withSuffix(index);- const target = this.withSuffix(index + 1);- if (NodeFS.existsSync(source)) {- NodeFS.renameSync(source, target);- }- }+ close(): Promise<void> {+ this.closed = true;+ return this.flush();+ }- if (NodeFS.existsSync(this.filePath)) {- NodeFS.renameSync(this.filePath, this.withSuffix(1));+ private async drain(): Promise<void> {+ let active: PendingChunk[] = [];+ try {+ await this.initialized;+ while (this.queue.length > 0) {+ if (this.fatalFailure) throw this.fatalFailure;+ const first = this.queue[0]!;+ if (this.currentSize > 0 && this.currentSize + first.buffer.length > this.maxBytes) {+ await this.rotate();+ }+ let bytes = first.buffer.length;+ let count = 1;+ while (count < this.queue.length) {+ const nextBytes = this.queue[count]!.buffer.length;+ if (+ bytes + nextBytes > MAX_APPEND_BATCH_BYTES ||+ this.currentSize + bytes + nextBytes > this.maxBytes+ )+ break;+ bytes += nextBytes;+ count += 1;+ }+ active = this.queue.splice(0, count);+ await NodeFSP.appendFile(+ this.filePath,+ count === 1+ ? first.buffer+ : Buffer.concat(+ active.map((entry) => entry.buffer),+ bytes,+ ),+ );+ this.currentSize += bytes;+ for (const entry of active) {+ this.pendingBytes -= entry.buffer.length;+ this.pendingChunks -= 1;+ entry.resolve();+ }+ active = [];}-- this.currentSize = 0;} catch (cause) {- if (this.throwOnError) {- throw new RotatingFileSinkError({- operation: "rotate",- filePath: this.filePath,- cause,- });- }- this.currentSize = this.readCurrentSize();+ const error = isRotatingFileSinkError(cause) ? cause : this.error("write", cause);+ // An append may have partially succeeded; never retry ambiguous bytes.+ this.failure ??= error;+ this.fatalFailure = error;+ for (const entry of [...active, ...this.queue.splice(0)]) entry.reject(error);+ this.pendingBytes = this.pendingChunks = 0;+ } finally {+ this.draining = false;}}- private pruneOverflowBackups(): void {+ private error(+ operation: RotatingFileSinkError["operation"],+ cause: unknown,+ ): RotatingFileSinkError {+ return new RotatingFileSinkError({ operation, filePath: this.filePath, cause });+ }++ private async initialize(): Promise<void> {+ try {+ await NodeFSP.mkdir(NodePath.dirname(this.filePath), { recursive: true });+ } catch (cause) {+ throw this.error("initialize", cause);+ }try {const dir = NodePath.dirname(this.filePath);const baseName = NodePath.basename(this.filePath);- for (const entry of NodeFS.readdirSync(dir)) {+ for (const entry of await NodeFSP.readdir(dir)) {if (!entry.startsWith(`${baseName}.`)) continue;const suffix = Number(entry.slice(baseName.length + 1));if (!Number.isInteger(suffix) || suffix <= this.maxFiles) continue;- NodeFS.rmSync(NodePath.join(dir, entry), { force: true });+ await NodeFSP.rm(NodePath.join(dir, entry), { force: true });}} catch (cause) {- if (this.throwOnError) {- throw new RotatingFileSinkError({- operation: "prune",- filePath: this.filePath,- cause,- });- }+ throw this.error("prune", cause);}- }-- private readCurrentSize(): number {try {- return NodeFS.statSync(this.filePath).size;+ this.currentSize = (await NodeFSP.stat(this.filePath)).size;} catch (cause) {- if (isFileNotFoundError(cause)) {- return 0;- }- throw new RotatingFileSinkError({- operation: "read",- filePath: this.filePath,- cause,- });+ if (!isFileNotFoundError(cause)) throw this.error("read", cause);}}- private withSuffix(index: number): string {- return `${this.filePath}.${index}`;+ private async rotate(): Promise<void> {+ try {+ await NodeFSP.rm(`${this.filePath}.${this.maxFiles}`, { force: true });+ for (let index = this.maxFiles - 1; index >= 0; index -= 1) {+ const source = index === 0 ? this.filePath : `${this.filePath}.${index}`;+ try {+ await NodeFSP.rename(source, `${this.filePath}.${index + 1}`);+ } catch (cause) {+ if (!isFileNotFoundError(cause)) throw cause;+ }+ }+ this.currentSize = 0;+ } catch (cause) {+ throw this.error("rotate", cause);+ }}}diff --git a/packages/shared/src/observability.test.ts b/packages/shared/src/observability.test.tsindex c58395393..9b817d53b 100644--- a/packages/shared/src/observability.test.ts+++ b/packages/shared/src/observability.test.ts@@ -2,7 +2,9 @@ import { assert, describe, it } from "@effect/vitest";import * as NodeServices from "@effect/platform-node/NodeServices";import * as Arr from "effect/Array";import * as Cause from "effect/Cause";+import * as Deferred from "effect/Deferred";import * as Effect from "effect/Effect";+import * as Fiber from "effect/Fiber";import * as FileSystem from "effect/FileSystem";import * as Layer from "effect/Layer";import * as Logger from "effect/Logger";@@ -204,6 +206,137 @@ describe("observability", () => {),);+ it.effect("drains trace records when the owning fiber is interrupted", () =>+ Effect.gen(function* () {+ const fileSystem = yield* FileSystem.FileSystem;+ const path = yield* Path.Path;+ const tempDir = yield* fileSystem.makeTempDirectoryScoped({+ prefix: "t3-trace-interrupt-",+ });+ for (const count of [0, 1, 256]) {+ const tracePath = path.join(tempDir, `${count}.ndjson`);+ const ready = yield* Deferred.make<void>();+ const owner = yield* Effect.gen(function* () {+ const sink = yield* makeTraceSink({+ filePath: tracePath,+ maxBytes: 1024 * 1024,+ maxFiles: 2,+ batchWindowMs: 10_000,+ });+ for (let index = 0; index < count; index++) sink.push(makeRecord(`record-${index}`));+ yield* Deferred.succeed(ready, undefined);+ return yield* Effect.never;+ }).pipe(Effect.scoped, Effect.forkChild);+ yield* Deferred.await(ready);+ yield* Fiber.interrupt(owner);+ if (count > 0) assert.equal((yield* readTraceRecords(tracePath)).length, count);+ }+ }),+ );++ it.effect("bounds queued trace bytes, reports drops, and rejects pushes after close", () =>+ Effect.scoped(+ Effect.gen(function* () {+ const fs = yield* FileSystem.FileSystem;+ const path = yield* Path.Path;+ const dir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-trace-bound-" });+ const tracePath = path.join(dir, "trace.ndjson");+ const messages: unknown[] = [];+ const capture = Logger.make<unknown, void>(({ message }) => {+ messages.push(message);+ });+ yield* Effect.gen(function* () {+ const sink = yield* makeTraceSink({+ filePath: tracePath,+ maxBytes: 1024,+ maxFiles: 2,+ maxBufferedBytes: 1024,+ batchWindowMs: 10_000,+ });+ for (let index = 0; index < 1000; index++) sink.push(makeRecord("bounded"));+ yield* sink.close();+ const records = yield* readTraceRecords(tracePath);+ assert.isAbove(records.length, 0);+ assert.isBelow(records.length, 1000);+ assert.isAtMost(Number((yield* fs.stat(tracePath)).size), 1024);+ sink.push(makeRecord("after-close"));+ yield* sink.flush;+ assert.equal((yield* readTraceRecords(tracePath)).length, records.length);+ }).pipe(Effect.provide(Logger.layer([capture], { mergeWithExisting: false })));+ assert.isAbove(messages.length, 0);+ }),+ ),+ );++ it.effect(+ "reports asynchronous write failures without retrying records or claiming attribution",+ () =>+ Effect.scoped(+ Effect.gen(function* () {+ const fs = yield* FileSystem.FileSystem;+ const path = yield* Path.Path;+ const dir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-trace-error-" });+ const tracePath = path.join(dir, "not-a-file");+ yield* fs.makeDirectory(tracePath);+ const messages: unknown[] = [];+ const stats: TraceSinkFlushStats[] = [];+ const capture = Logger.make<unknown, void>(({ message }) => {+ messages.push(message);+ });+ yield* Effect.gen(function* () {+ const sink = yield* makeTraceSink({+ filePath: tracePath,+ maxBytes: 1024 * 1024,+ maxFiles: 2,+ batchWindowMs: 10_000,+ onFlush: (entry) =>+ Effect.sync(() => {+ stats.push(entry);+ }),+ });+ sink.push(makeRecord("failed"));+ yield* sink.flush;+ yield* sink.close();+ yield* sink.close();+ }).pipe(Effect.provide(Logger.layer([capture], { mergeWithExisting: false })));+ assert.equal(messages.length, 1);+ assert.deepEqual(stats, []);+ }),+ ),+ );++ it.effect(+ "honors the local tracer buffer limit and flushes its owned sink on scope close",+ () =>+ Effect.gen(function* () {+ const fs = yield* FileSystem.FileSystem;+ const path = yield* Path.Path;+ const dir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-local-tracer-bound-" });+ const tracePath = path.join(dir, "trace.ndjson");+ yield* Effect.scoped(+ Effect.gen(function* () {+ const tracer = yield* makeLocalFileTracer({+ filePath: tracePath,+ maxBytes: 1024 * 1024,+ maxFiles: 2,+ maxBufferedBytes: 1024,+ batchWindowMs: 10_000,+ });+ for (let index = 0; index < 100; index++) {+ yield* Effect.void.pipe(+ Effect.withSpan("bounded"),+ Effect.provideService(Tracer.Tracer, tracer),+ );+ }+ }),+ );+ const records = yield* readTraceRecords(tracePath);+ assert.isAbove(records.length, 0);+ assert.isBelow(records.length, 100);+ assert.isAtMost(Number((yield* fs.stat(tracePath)).size), 1024);+ }),+ );+it.effect("reports successful logical trace writes", () =>Effect.scoped(Effect.gen(function* () {diff --git a/packages/shared/src/observability.ts b/packages/shared/src/observability.tsindex 67057c548..7929803fe 100644--- a/packages/shared/src/observability.ts+++ b/packages/shared/src/observability.ts@@ -9,7 +9,7 @@ import { OtlpResource, OtlpTracer } from "effect/unstable/observability";import { RotatingFileSink } from "./logging.ts";const FLUSH_BUFFER_THRESHOLD = 256;-const textEncoder = new TextEncoder();+const DEFAULT_MAX_BUFFERED_BYTES = 1024 * 1024;export type TraceAttributes = Readonly<Record<string, unknown>>;@@ -110,6 +110,7 @@ export interface TraceSinkOptions {readonly maxBytes: number;readonly maxFiles: number;readonly batchWindowMs: number;+ readonly maxBufferedBytes?: number;readonly onFlush?: (stats: TraceSinkFlushStats) => Effect.Effect<void>;}@@ -336,79 +337,85 @@ export function spanToTraceRecord(span: SerializableSpan): EffectTraceRecord {}export const makeTraceSink = Effect.fn("makeTraceSink")(function* (options: TraceSinkOptions) {+ const maxBufferedBytes = options.maxBufferedBytes ?? DEFAULT_MAX_BUFFERED_BYTES;const sink = new RotatingFileSink({filePath: options.filePath,maxBytes: options.maxBytes,maxFiles: options.maxFiles,- throwOnError: true,+ maxBufferedBytes,});- let buffer: Array<string> = [];- let pendingFlushStats: TraceSinkFlushStats = {- logicalWriteBytes: 0,- count: 0,- durationMs: 0,- };-- const flushUnsafe = () => {- if (buffer.length === 0) {- return;- }+ let buffer: Array<{ line: string; bytes: number }> = [];+ let bufferedBytes = 0;+ let closed = false;+ let droppedRecords = 0;+ let writeError: unknown;+ let reportedWriteError = false;+ let pendingFlushStats: TraceSinkFlushStats = { logicalWriteBytes: 0, count: 0, durationMs: 0 };+ const submit = () => {const records = buffer;buffer = [];- let persistedCount = 0;-- while (persistedCount < records.length) {- const firstRecordBytes = textEncoder.encode(records[persistedCount]).byteLength;- if (firstRecordBytes > options.maxBytes) {- persistedCount += 1;- continue;+ bufferedBytes = 0;+ let index = 0;+ while (index < records.length) {+ const start = index;+ let bytes = 0;+ const lines: string[] = [];+ while (index < records.length) {+ const record = records[index]!;+ if (bytes + record.bytes > options.maxBytes) break;+ bytes += record.bytes;+ lines.push(record.line);+ index += 1;}-- let nextIndex = persistedCount + 1;- let chunkBytes = firstRecordBytes;- while (nextIndex < records.length) {- const nextRecordBytes = textEncoder.encode(records[nextIndex]).byteLength;- if (chunkBytes + nextRecordBytes > options.maxBytes) break;- chunkBytes += nextRecordBytes;- nextIndex += 1;- }-- const chunk = records.slice(persistedCount, nextIndex).join("");+ const count = index - start;const startedAt = performance.now();- try {- sink.write(chunk);- } catch {- buffer.unshift(...records.slice(persistedCount));- return;- }- pendingFlushStats = {- logicalWriteBytes: pendingFlushStats.logicalWriteBytes + chunkBytes,- count: pendingFlushStats.count + nextIndex - persistedCount,- durationMs: pendingFlushStats.durationMs + Math.max(0, performance.now() - startedAt),- };- persistedCount = nextIndex;+ void sink.write(lines.join("")).then(+ () => {+ pendingFlushStats = {+ logicalWriteBytes: pendingFlushStats.logicalWriteBytes + bytes,+ count: pendingFlushStats.count + count,+ durationMs: pendingFlushStats.durationMs + Math.max(0, performance.now() - startedAt),+ };+ },+ (cause: unknown) => {+ writeError = cause;+ droppedRecords += count;+ },+ );}};- const flush = Effect.sync(() => {- flushUnsafe();- const stats = pendingFlushStats;- pendingFlushStats = {- logicalWriteBytes: 0,- count: 0,- durationMs: 0,- };- return stats;- }).pipe(- Effect.flatMap((stats) =>- stats.count > 0 && options.onFlush ? options.onFlush(stats).pipe(Effect.ignore) : Effect.void,- ),- Effect.withTracerEnabled(false),- );+ const drain = (close: boolean) =>+ Effect.gen(function* () {+ if (close) closed = true;+ submit();+ yield* Effect.promise(() => (close ? sink.close() : sink.flush())).pipe(+ Effect.catchCause((cause) =>+ Effect.sync(() => {+ writeError ??= Cause.squash(cause);+ }),+ ),+ );+ const stats = pendingFlushStats;+ pendingFlushStats = { logicalWriteBytes: 0, count: 0, durationMs: 0 };+ const dropped = droppedRecords;+ droppedRecords = 0;+ if (dropped > 0 || (writeError !== undefined && !reportedWriteError)) {+ reportedWriteError = writeError !== undefined;+ yield* Effect.logWarning("trace log records could not be persisted", {+ filePath: options.filePath,+ droppedRecords: dropped,+ ...(writeError !== undefined ? { errorTag: errorTag(writeError) } : {}),+ });+ }+ if (stats.count > 0 && options.onFlush) yield* options.onFlush(stats).pipe(Effect.ignore);+ }).pipe(Effect.withTracerEnabled(false), Effect.uninterruptible);+ const flush = drain(false);+ const close = () => drain(true);- yield* Effect.addFinalizer(() => flush.pipe(Effect.ignore));+ yield* Effect.addFinalizer(close);yield* Effect.forkScoped(Effect.sleep(`${options.batchWindowMs} millis`).pipe(Effect.andThen(flush), Effect.forever),);@@ -416,17 +423,26 @@ export const makeTraceSink = Effect.fn("makeTraceSink")(function* (options: Tracreturn {filePath: options.filePath,push(record) {+ if (closed) return;try {- buffer.push(`${JSON.stringify(record)}\n`);- if (buffer.length >= FLUSH_BUFFER_THRESHOLD) {- flushUnsafe();+ const line = `${JSON.stringify(record)}\n`;+ const bytes = Buffer.byteLength(line);+ if (+ bytes > options.maxBytes ||+ bufferedBytes + sink.bufferedBytes + bytes > maxBufferedBytes+ ) {+ droppedRecords += 1;+ return;}+ buffer.push({ line, bytes });+ bufferedBytes += bytes;+ if (buffer.length >= FLUSH_BUFFER_THRESHOLD) submit();} catch {- return;+ droppedRecords += 1;}},flush,- close: () => flush,+ close,} satisfies TraceSink;});@@ -511,6 +527,9 @@ export const makeLocalFileTracer = Effect.fn("makeLocalFileTracer")(function* (maxBytes: options.maxBytes,maxFiles: options.maxFiles,batchWindowMs: options.batchWindowMs,+ ...(options.maxBufferedBytes !== undefined+ ? { maxBufferedBytes: options.maxBufferedBytes }+ : {}),...(options.onFlush ? { onFlush: options.onFlush } : {}),}));