Skip to content

Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.

feat(server): agents can now open the images you paste into chat (#5757)

Theo Brownea6c9b41f902f
diff --git a/apps/server/integration/providerService.integration.test.ts b/apps/server/integration/providerService.integration.test.ts
index c57d289f9..6089d22d9 100644
--- a/apps/server/integration/providerService.integration.test.ts
+++ b/apps/server/integration/providerService.integration.test.ts
@@ -23,6 +23,7 @@ import {
ProviderService,
type ProviderServiceShape,
} from "../src/provider/Services/ProviderService.ts";
+import * as ServerConfig from "../src/config.ts";
import { ServerSettingsService } from "../src/serverSettings.ts";
import { AnalyticsService } from "../src/telemetry/Services/AnalyticsService.ts";
import { SqlitePersistenceMemory } from "../src/persistence/Layers/Sqlite.ts";
@@ -93,6 +94,7 @@ const makeIntegrationFixture = (options?: { readonly analytics?: Layer.Layer<Ana
const shared = Layer.mergeAll(
directoryLayer,
Layer.succeed(ProviderAdapterRegistry, registry),
+ ServerConfig.layerTest(cwd, cwd).pipe(Layer.provide(NodeServices.layer)),
ServerSettingsService.layerTest(DEFAULT_SERVER_SETTINGS),
options?.analytics ?? AnalyticsService.layerTest,
Layer.succeed(ProviderEventLoggers, NoOpProviderEventLoggers),
diff --git a/apps/server/src/provider/Layers/ClaudeAdapter.ts b/apps/server/src/provider/Layers/ClaudeAdapter.ts
index 00839c455..b1fa74e81 100644
--- a/apps/server/src/provider/Layers/ClaudeAdapter.ts
+++ b/apps/server/src/provider/Layers/ClaudeAdapter.ts
@@ -4090,6 +4090,14 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* (
...(ultracode ? { ultracode: true } : {}),
};
const mcpSession = McpProviderSession.readMcpProviderSession(input.threadId);
+ // The attachments dir grant lets the agent Read/copy pasted images at
+ // the paths ProviderService injects into the turn text, without an
+ // approval prompt. It is a leaf directory holding only attachment
+ // files; siblings like secrets/ and state.sqlite stay ungranted.
+ const additionalDirectories = [
+ ...(input.cwd ? [input.cwd] : []),
+ serverConfig.attachmentsDir,
+ ];
const queryOptions: ClaudeQueryOptions = {
...(input.cwd ? { cwd: input.cwd } : {}),
...(apiModelId ? { model: apiModelId } : {}),
@@ -4113,7 +4121,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* (
includePartialMessages: true,
canUseTool,
env: claudeEnvironment,
- ...(input.cwd ? { additionalDirectories: [input.cwd] } : {}),
+ additionalDirectories,
...(Object.keys(extraArgs).length > 0 ? { extraArgs } : {}),
...(mcpSession
? {
@@ -4148,7 +4156,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* (
"claude.query.resume": existingResumeSessionId ?? "",
"claude.query.session_id": newSessionId ?? "",
"claude.query.include_partial_messages": true,
- "claude.query.additional_directories": input.cwd ? [input.cwd] : [],
+ "claude.query.additional_directories": additionalDirectories,
"claude.query.setting_sources": [...CLAUDE_SETTING_SOURCES],
"claude.query.settings_json": encodeJsonStringForDiagnostics(settings) ?? "",
"claude.query.extra_args_json": encodeJsonStringForDiagnostics(extraArgs) ?? "",
diff --git a/apps/server/src/provider/Layers/ProviderService.test.ts b/apps/server/src/provider/Layers/ProviderService.test.ts
index ccbbce175..7334cd019 100644
--- a/apps/server/src/provider/Layers/ProviderService.test.ts
+++ b/apps/server/src/provider/Layers/ProviderService.test.ts
@@ -55,11 +55,15 @@ import {
makeSqlitePersistenceLive,
SqlitePersistenceMemory,
} from "../../persistence/Layers/Sqlite.ts";
+import * as ServerConfig from "../../config.ts";
import * as ServerSettings from "../../serverSettings.ts";
import * as AnalyticsService from "../../telemetry/AnalyticsService.ts";
import { makeAdapterRegistryMock } from "../testUtils/providerAdapterRegistryMock.ts";
const defaultServerSettingsLayer = ServerSettings.ServerSettingsService.layerTest();
+const serverConfigTestLayer = ServerConfig.layerTest(process.cwd(), process.cwd()).pipe(
+ Layer.provide(NodeServices.layer),
+);
const asRequestId = (value: string): ApprovalRequestId => ApprovalRequestId.make(value);
const asEventId = (value: string): EventId => EventId.make(value);
@@ -292,6 +296,7 @@ function makeProviderServiceLayer() {
Layer.provide(providerAdapterLayer),
Layer.provide(directoryLayer),
Layer.provide(defaultServerSettingsLayer),
+ Layer.provide(serverConfigTestLayer),
Layer.provideMerge(AnalyticsService.layerTest),
Layer.provide(
Layer.succeed(
@@ -343,6 +348,7 @@ it.effect("ProviderServiceLive catches stopAll failures during shutdown", () =>
Layer.provide(providerAdapterLayer),
Layer.provide(directoryLayer),
Layer.provide(defaultServerSettingsLayer),
+ Layer.provide(serverConfigTestLayer),
Layer.provideMerge(AnalyticsService.layerTest),
Layer.provide(
Layer.succeed(
@@ -402,6 +408,7 @@ it.effect("ProviderServiceLive rejects new sessions for disabled providers", ()
Layer.provide(providerAdapterLayer),
Layer.provide(directoryLayer),
Layer.provide(defaultServerSettingsLayer),
+ Layer.provide(serverConfigTestLayer),
Layer.provide(AnalyticsService.layerTest),
Layer.provide(
Layer.succeed(
@@ -486,6 +493,7 @@ it.effect(
Layer.provide(providerAdapterLayer),
Layer.provide(directoryLayer),
Layer.provide(serverSettingsLayer),
+ Layer.provide(serverConfigTestLayer),
Layer.provide(AnalyticsService.layerTest),
Layer.provide(
Layer.succeed(
@@ -556,6 +564,7 @@ it.effect("ProviderServiceLive rejects new sessions for disabled custom instance
Layer.provide(providerAdapterLayer),
Layer.provide(directoryLayer),
Layer.provide(defaultServerSettingsLayer),
+ Layer.provide(serverConfigTestLayer),
Layer.provide(AnalyticsService.layerTest),
Layer.provide(
Layer.succeed(
@@ -611,6 +620,7 @@ it.effect("ProviderServiceLive writes canonical events to the emitting thread se
Layer.provide(Layer.succeed(ProviderAdapterRegistry.ProviderAdapterRegistry, registry)),
Layer.provide(directoryLayer),
Layer.provide(defaultServerSettingsLayer),
+ Layer.provide(serverConfigTestLayer),
Layer.provide(AnalyticsService.layerTest),
Layer.provide(
Layer.succeed(
@@ -671,6 +681,7 @@ it.effect("ProviderServiceLive keeps persisted resumable sessions on startup", (
Layer.provide(Layer.succeed(ProviderAdapterRegistry.ProviderAdapterRegistry, registry)),
Layer.provide(directoryLayer),
Layer.provide(defaultServerSettingsLayer),
+ Layer.provide(serverConfigTestLayer),
Layer.provide(AnalyticsService.layerTest),
Layer.provide(
Layer.succeed(
@@ -737,6 +748,7 @@ it.effect(
),
Layer.provide(firstDirectoryLayer),
Layer.provide(defaultServerSettingsLayer),
+ Layer.provide(serverConfigTestLayer),
Layer.provide(AnalyticsService.layerTest),
Layer.provide(
Layer.succeed(
@@ -796,6 +808,7 @@ it.effect(
),
Layer.provide(secondDirectoryLayer),
Layer.provide(defaultServerSettingsLayer),
+ Layer.provide(serverConfigTestLayer),
Layer.provide(AnalyticsService.layerTest),
Layer.provide(
Layer.succeed(
@@ -927,6 +940,54 @@ routing.layer("ProviderServiceLive routing", (it) => {
}),
);
+ it.effect("appends attachment file paths to the turn input text", () =>
+ Effect.gen(function* () {
+ const provider = yield* ProviderService.ProviderService;
+
+ const session = yield* provider.startSession(asThreadId("thread-attach"), {
+ provider: ProviderDriverKind.make("codex"),
+ providerInstanceId: codexInstanceId,
+ threadId: asThreadId("thread-attach"),
+ cwd: "/tmp/project",
+ runtimeMode: "full-access",
+ });
+
+ const attachment = {
+ type: "image" as const,
+ id: "thread-attach-12345678-1234-1234-1234-123456789abc",
+ name: "screenshot.png",
+ mimeType: "image/png",
+ sizeBytes: 123,
+ };
+
+ routing.codex.sendTurn.mockClear();
+ yield* provider.sendTurn({
+ threadId: session.threadId,
+ input: "use this screenshot",
+ attachments: [attachment],
+ });
+
+ const turnInput = routing.codex.sendTurn.mock.calls[0]?.[0] as ProviderSendTurnInput;
+ assert.equal(typeof turnInput.input, "string");
+ const turnText = turnInput.input ?? "";
+ assert.equal(turnText.startsWith("use this screenshot"), true);
+ assert.include(turnText, '[Attached image "screenshot.png" is saved at: ');
+ assert.equal(turnText.endsWith(`${attachment.id}.png]`), true);
+
+ // An attachment-only turn stays valid and the injected line becomes the
+ // whole input text, so the agent still learns the path.
+ routing.codex.sendTurn.mockClear();
+ yield* provider.sendTurn({
+ threadId: session.threadId,
+ attachments: [attachment],
+ });
+ const imageOnlyInput = routing.codex.sendTurn.mock.calls[0]?.[0] as ProviderSendTurnInput;
+ assert.equal(imageOnlyInput.input?.startsWith('[Attached image "screenshot.png"'), true);
+
+ yield* provider.stopSession({ threadId: session.threadId });
+ }),
+ );
+
it.effect("recovers stale persisted sessions for rollback by resuming thread identity", () =>
Effect.gen(function* () {
const provider = yield* ProviderService.ProviderService;
@@ -1307,6 +1368,7 @@ routing.layer("ProviderServiceLive routing", (it) => {
),
Layer.provide(firstDirectoryLayer),
Layer.provide(defaultServerSettingsLayer),
+ Layer.provide(serverConfigTestLayer),
Layer.provide(AnalyticsService.layerTest),
Layer.provide(
Layer.succeed(
@@ -1345,6 +1407,7 @@ routing.layer("ProviderServiceLive routing", (it) => {
),
Layer.provide(secondDirectoryLayer),
Layer.provide(defaultServerSettingsLayer),
+ Layer.provide(serverConfigTestLayer),
Layer.provide(AnalyticsService.layerTest),
Layer.provide(
Layer.succeed(
@@ -1413,6 +1476,7 @@ routing.layer("ProviderServiceLive routing", (it) => {
),
Layer.provide(firstDirectoryLayer),
Layer.provide(defaultServerSettingsLayer),
+ Layer.provide(serverConfigTestLayer),
Layer.provide(AnalyticsService.layerTest),
Layer.provide(
Layer.succeed(
@@ -1446,6 +1510,7 @@ routing.layer("ProviderServiceLive routing", (it) => {
),
Layer.provide(secondDirectoryLayer),
Layer.provide(defaultServerSettingsLayer),
+ Layer.provide(serverConfigTestLayer),
Layer.provide(AnalyticsService.layerTest),
Layer.provide(
Layer.succeed(
diff --git a/apps/server/src/provider/Layers/ProviderService.ts b/apps/server/src/provider/Layers/ProviderService.ts
index d0acc1039..2ac00873d 100644
--- a/apps/server/src/provider/Layers/ProviderService.ts
+++ b/apps/server/src/provider/Layers/ProviderService.ts
@@ -35,6 +35,8 @@ import * as Schema from "effect/Schema";
import * as SchemaIssue from "effect/SchemaIssue";
import * as Stream from "effect/Stream";
+import { resolveAttachmentPath } from "../../attachmentStore.ts";
+import * as ServerConfig from "../../config.ts";
import {
increment,
providerMetricAttributes,
@@ -203,6 +205,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
options?: ProviderServiceLiveOptions,
) {
const analytics = yield* Effect.service(AnalyticsService.AnalyticsService);
+ const serverConfig = yield* ServerConfig.ServerConfig;
const eventLoggers = yield* ProviderEventLoggers.ProviderEventLoggers;
// Options-provided logger wins (test overrides); otherwise we take whatever
// the `ProviderEventLoggers` tag exposes — `undefined` means "no canonical
@@ -665,16 +668,44 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
payload: rawInput,
});
- const input = {
- ...parsed,
- attachments: parsed.attachments ?? [],
- };
- if (!input.input && input.attachments.length === 0) {
+ const attachments = parsed.attachments ?? [];
+ if (!parsed.input && attachments.length === 0) {
return yield* toValidationError(
"ProviderService.sendTurn",
"Either input text or at least one attachment is required",
);
}
+
+ // Adapters inline attachment pixels into the model prompt, but the model's
+ // tools cannot dereference pixels. Appending the on-disk path is what lets
+ // a turn like "include this screenshot in the PR" copy the actual file.
+ // This runs after schema decode, so the appended lines are exempt from the
+ // PROVIDER_SEND_TURN_MAX_INPUT_CHARS check; attachment count is capped, so
+ // the overhead is bounded. Unresolvable ids are skipped here and surface
+ // as adapter errors when the file is read for inlining.
+ const attachmentPathLines = attachments.flatMap((attachment) => {
+ const attachmentPath = resolveAttachmentPath({
+ attachmentsDir: serverConfig.attachmentsDir,
+ attachment,
+ });
+ return attachmentPath === null
+ ? []
+ : [`[Attached ${attachment.type} "${attachment.name}" is saved at: ${attachmentPath}]`];
+ });
+ const inputTextWithAttachmentPaths =
+ attachmentPathLines.length === 0
+ ? parsed.input
+ : [parsed.input, attachmentPathLines.join("\n")]
+ .filter((part): part is string => typeof part === "string" && part.length > 0)
+ .join("\n\n");
+
+ const input = {
+ ...parsed,
+ ...(inputTextWithAttachmentPaths !== undefined
+ ? { input: inputTextWithAttachmentPaths }
+ : {}),
+ attachments,
+ };
yield* Effect.annotateCurrentSpan({
"provider.operation": "send-turn",
"provider.thread_id": input.threadId,