Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.
feat(server): agents can now open the images you paste into chat (#5757)
Theo Brownea6c9b41f902f
diff --git a/apps/server/integration/providerService.integration.test.ts b/apps/server/integration/providerService.integration.test.tsindex c57d289f9..6089d22d9 100644--- a/apps/server/integration/providerService.integration.test.ts+++ b/apps/server/integration/providerService.integration.test.ts@@ -23,6 +23,7 @@ import {ProviderService,type ProviderServiceShape,} from "../src/provider/Services/ProviderService.ts";+import * as ServerConfig from "../src/config.ts";import { ServerSettingsService } from "../src/serverSettings.ts";import { AnalyticsService } from "../src/telemetry/Services/AnalyticsService.ts";import { SqlitePersistenceMemory } from "../src/persistence/Layers/Sqlite.ts";@@ -93,6 +94,7 @@ const makeIntegrationFixture = (options?: { readonly analytics?: Layer.Layer<Anaconst shared = Layer.mergeAll(directoryLayer,Layer.succeed(ProviderAdapterRegistry, registry),+ ServerConfig.layerTest(cwd, cwd).pipe(Layer.provide(NodeServices.layer)),ServerSettingsService.layerTest(DEFAULT_SERVER_SETTINGS),options?.analytics ?? AnalyticsService.layerTest,Layer.succeed(ProviderEventLoggers, NoOpProviderEventLoggers),diff --git a/apps/server/src/provider/Layers/ClaudeAdapter.ts b/apps/server/src/provider/Layers/ClaudeAdapter.tsindex 00839c455..b1fa74e81 100644--- a/apps/server/src/provider/Layers/ClaudeAdapter.ts+++ b/apps/server/src/provider/Layers/ClaudeAdapter.ts@@ -4090,6 +4090,14 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* (...(ultracode ? { ultracode: true } : {}),};const mcpSession = McpProviderSession.readMcpProviderSession(input.threadId);+ // The attachments dir grant lets the agent Read/copy pasted images at+ // the paths ProviderService injects into the turn text, without an+ // approval prompt. It is a leaf directory holding only attachment+ // files; siblings like secrets/ and state.sqlite stay ungranted.+ const additionalDirectories = [+ ...(input.cwd ? [input.cwd] : []),+ serverConfig.attachmentsDir,+ ];const queryOptions: ClaudeQueryOptions = {...(input.cwd ? { cwd: input.cwd } : {}),...(apiModelId ? { model: apiModelId } : {}),@@ -4113,7 +4121,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* (includePartialMessages: true,canUseTool,env: claudeEnvironment,- ...(input.cwd ? { additionalDirectories: [input.cwd] } : {}),+ additionalDirectories,...(Object.keys(extraArgs).length > 0 ? { extraArgs } : {}),...(mcpSession? {@@ -4148,7 +4156,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ("claude.query.resume": existingResumeSessionId ?? "","claude.query.session_id": newSessionId ?? "","claude.query.include_partial_messages": true,- "claude.query.additional_directories": input.cwd ? [input.cwd] : [],+ "claude.query.additional_directories": additionalDirectories,"claude.query.setting_sources": [...CLAUDE_SETTING_SOURCES],"claude.query.settings_json": encodeJsonStringForDiagnostics(settings) ?? "","claude.query.extra_args_json": encodeJsonStringForDiagnostics(extraArgs) ?? "",diff --git a/apps/server/src/provider/Layers/ProviderService.test.ts b/apps/server/src/provider/Layers/ProviderService.test.tsindex ccbbce175..7334cd019 100644--- a/apps/server/src/provider/Layers/ProviderService.test.ts+++ b/apps/server/src/provider/Layers/ProviderService.test.ts@@ -55,11 +55,15 @@ import {makeSqlitePersistenceLive,SqlitePersistenceMemory,} from "../../persistence/Layers/Sqlite.ts";+import * as ServerConfig from "../../config.ts";import * as ServerSettings from "../../serverSettings.ts";import * as AnalyticsService from "../../telemetry/AnalyticsService.ts";import { makeAdapterRegistryMock } from "../testUtils/providerAdapterRegistryMock.ts";const defaultServerSettingsLayer = ServerSettings.ServerSettingsService.layerTest();+const serverConfigTestLayer = ServerConfig.layerTest(process.cwd(), process.cwd()).pipe(+ Layer.provide(NodeServices.layer),+);const asRequestId = (value: string): ApprovalRequestId => ApprovalRequestId.make(value);const asEventId = (value: string): EventId => EventId.make(value);@@ -292,6 +296,7 @@ function makeProviderServiceLayer() {Layer.provide(providerAdapterLayer),Layer.provide(directoryLayer),Layer.provide(defaultServerSettingsLayer),+ Layer.provide(serverConfigTestLayer),Layer.provideMerge(AnalyticsService.layerTest),Layer.provide(Layer.succeed(@@ -343,6 +348,7 @@ it.effect("ProviderServiceLive catches stopAll failures during shutdown", () =>Layer.provide(providerAdapterLayer),Layer.provide(directoryLayer),Layer.provide(defaultServerSettingsLayer),+ Layer.provide(serverConfigTestLayer),Layer.provideMerge(AnalyticsService.layerTest),Layer.provide(Layer.succeed(@@ -402,6 +408,7 @@ it.effect("ProviderServiceLive rejects new sessions for disabled providers", ()Layer.provide(providerAdapterLayer),Layer.provide(directoryLayer),Layer.provide(defaultServerSettingsLayer),+ Layer.provide(serverConfigTestLayer),Layer.provide(AnalyticsService.layerTest),Layer.provide(Layer.succeed(@@ -486,6 +493,7 @@ it.effect(Layer.provide(providerAdapterLayer),Layer.provide(directoryLayer),Layer.provide(serverSettingsLayer),+ Layer.provide(serverConfigTestLayer),Layer.provide(AnalyticsService.layerTest),Layer.provide(Layer.succeed(@@ -556,6 +564,7 @@ it.effect("ProviderServiceLive rejects new sessions for disabled custom instanceLayer.provide(providerAdapterLayer),Layer.provide(directoryLayer),Layer.provide(defaultServerSettingsLayer),+ Layer.provide(serverConfigTestLayer),Layer.provide(AnalyticsService.layerTest),Layer.provide(Layer.succeed(@@ -611,6 +620,7 @@ it.effect("ProviderServiceLive writes canonical events to the emitting thread seLayer.provide(Layer.succeed(ProviderAdapterRegistry.ProviderAdapterRegistry, registry)),Layer.provide(directoryLayer),Layer.provide(defaultServerSettingsLayer),+ Layer.provide(serverConfigTestLayer),Layer.provide(AnalyticsService.layerTest),Layer.provide(Layer.succeed(@@ -671,6 +681,7 @@ it.effect("ProviderServiceLive keeps persisted resumable sessions on startup", (Layer.provide(Layer.succeed(ProviderAdapterRegistry.ProviderAdapterRegistry, registry)),Layer.provide(directoryLayer),Layer.provide(defaultServerSettingsLayer),+ Layer.provide(serverConfigTestLayer),Layer.provide(AnalyticsService.layerTest),Layer.provide(Layer.succeed(@@ -737,6 +748,7 @@ it.effect(),Layer.provide(firstDirectoryLayer),Layer.provide(defaultServerSettingsLayer),+ Layer.provide(serverConfigTestLayer),Layer.provide(AnalyticsService.layerTest),Layer.provide(Layer.succeed(@@ -796,6 +808,7 @@ it.effect(),Layer.provide(secondDirectoryLayer),Layer.provide(defaultServerSettingsLayer),+ Layer.provide(serverConfigTestLayer),Layer.provide(AnalyticsService.layerTest),Layer.provide(Layer.succeed(@@ -927,6 +940,54 @@ routing.layer("ProviderServiceLive routing", (it) => {}),);+ it.effect("appends attachment file paths to the turn input text", () =>+ Effect.gen(function* () {+ const provider = yield* ProviderService.ProviderService;++ const session = yield* provider.startSession(asThreadId("thread-attach"), {+ provider: ProviderDriverKind.make("codex"),+ providerInstanceId: codexInstanceId,+ threadId: asThreadId("thread-attach"),+ cwd: "/tmp/project",+ runtimeMode: "full-access",+ });++ const attachment = {+ type: "image" as const,+ id: "thread-attach-12345678-1234-1234-1234-123456789abc",+ name: "screenshot.png",+ mimeType: "image/png",+ sizeBytes: 123,+ };++ routing.codex.sendTurn.mockClear();+ yield* provider.sendTurn({+ threadId: session.threadId,+ input: "use this screenshot",+ attachments: [attachment],+ });++ const turnInput = routing.codex.sendTurn.mock.calls[0]?.[0] as ProviderSendTurnInput;+ assert.equal(typeof turnInput.input, "string");+ const turnText = turnInput.input ?? "";+ assert.equal(turnText.startsWith("use this screenshot"), true);+ assert.include(turnText, '[Attached image "screenshot.png" is saved at: ');+ assert.equal(turnText.endsWith(`${attachment.id}.png]`), true);++ // An attachment-only turn stays valid and the injected line becomes the+ // whole input text, so the agent still learns the path.+ routing.codex.sendTurn.mockClear();+ yield* provider.sendTurn({+ threadId: session.threadId,+ attachments: [attachment],+ });+ const imageOnlyInput = routing.codex.sendTurn.mock.calls[0]?.[0] as ProviderSendTurnInput;+ assert.equal(imageOnlyInput.input?.startsWith('[Attached image "screenshot.png"'), true);++ yield* provider.stopSession({ threadId: session.threadId });+ }),+ );+it.effect("recovers stale persisted sessions for rollback by resuming thread identity", () =>Effect.gen(function* () {const provider = yield* ProviderService.ProviderService;@@ -1307,6 +1368,7 @@ routing.layer("ProviderServiceLive routing", (it) => {),Layer.provide(firstDirectoryLayer),Layer.provide(defaultServerSettingsLayer),+ Layer.provide(serverConfigTestLayer),Layer.provide(AnalyticsService.layerTest),Layer.provide(Layer.succeed(@@ -1345,6 +1407,7 @@ routing.layer("ProviderServiceLive routing", (it) => {),Layer.provide(secondDirectoryLayer),Layer.provide(defaultServerSettingsLayer),+ Layer.provide(serverConfigTestLayer),Layer.provide(AnalyticsService.layerTest),Layer.provide(Layer.succeed(@@ -1413,6 +1476,7 @@ routing.layer("ProviderServiceLive routing", (it) => {),Layer.provide(firstDirectoryLayer),Layer.provide(defaultServerSettingsLayer),+ Layer.provide(serverConfigTestLayer),Layer.provide(AnalyticsService.layerTest),Layer.provide(Layer.succeed(@@ -1446,6 +1510,7 @@ routing.layer("ProviderServiceLive routing", (it) => {),Layer.provide(secondDirectoryLayer),Layer.provide(defaultServerSettingsLayer),+ Layer.provide(serverConfigTestLayer),Layer.provide(AnalyticsService.layerTest),Layer.provide(Layer.succeed(diff --git a/apps/server/src/provider/Layers/ProviderService.ts b/apps/server/src/provider/Layers/ProviderService.tsindex d0acc1039..2ac00873d 100644--- a/apps/server/src/provider/Layers/ProviderService.ts+++ b/apps/server/src/provider/Layers/ProviderService.ts@@ -35,6 +35,8 @@ import * as Schema from "effect/Schema";import * as SchemaIssue from "effect/SchemaIssue";import * as Stream from "effect/Stream";+import { resolveAttachmentPath } from "../../attachmentStore.ts";+import * as ServerConfig from "../../config.ts";import {increment,providerMetricAttributes,@@ -203,6 +205,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (options?: ProviderServiceLiveOptions,) {const analytics = yield* Effect.service(AnalyticsService.AnalyticsService);+ const serverConfig = yield* ServerConfig.ServerConfig;const eventLoggers = yield* ProviderEventLoggers.ProviderEventLoggers;// Options-provided logger wins (test overrides); otherwise we take whatever// the `ProviderEventLoggers` tag exposes — `undefined` means "no canonical@@ -665,16 +668,44 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (payload: rawInput,});- const input = {- ...parsed,- attachments: parsed.attachments ?? [],- };- if (!input.input && input.attachments.length === 0) {+ const attachments = parsed.attachments ?? [];+ if (!parsed.input && attachments.length === 0) {return yield* toValidationError("ProviderService.sendTurn","Either input text or at least one attachment is required",);}++ // Adapters inline attachment pixels into the model prompt, but the model's+ // tools cannot dereference pixels. Appending the on-disk path is what lets+ // a turn like "include this screenshot in the PR" copy the actual file.+ // This runs after schema decode, so the appended lines are exempt from the+ // PROVIDER_SEND_TURN_MAX_INPUT_CHARS check; attachment count is capped, so+ // the overhead is bounded. Unresolvable ids are skipped here and surface+ // as adapter errors when the file is read for inlining.+ const attachmentPathLines = attachments.flatMap((attachment) => {+ const attachmentPath = resolveAttachmentPath({+ attachmentsDir: serverConfig.attachmentsDir,+ attachment,+ });+ return attachmentPath === null+ ? []+ : [`[Attached ${attachment.type} "${attachment.name}" is saved at: ${attachmentPath}]`];+ });+ const inputTextWithAttachmentPaths =+ attachmentPathLines.length === 0+ ? parsed.input+ : [parsed.input, attachmentPathLines.join("\n")]+ .filter((part): part is string => typeof part === "string" && part.length > 0)+ .join("\n\n");++ const input = {+ ...parsed,+ ...(inputTextWithAttachmentPaths !== undefined+ ? { input: inputTextWithAttachmentPaths }+ : {}),+ attachments,+ };yield* Effect.annotateCurrentSpan({"provider.operation": "send-turn","provider.thread_id": input.threadId,