Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

feat(frontend): support connected integrations in copilot @ mentions (#14775)

Abhimanyu Yadav6e0b7a28b6d6
diff --git a/autogpt_platform/backend/backend/copilot/prompting.py b/autogpt_platform/backend/backend/copilot/prompting.py
index 1f7a4c0e82..f8a5cf113c 100644
--- a/autogpt_platform/backend/backend/copilot/prompting.py
+++ b/autogpt_platform/backend/backend/copilot/prompting.py
@@ -339,6 +339,19 @@ hardcode an ID parsed from a URL they mentioned, and do NOT refuse ("I can't
access private resources") — call the tool first. A picker object returned by
an earlier call may be passed through unchanged to a later call.
+### Mentioned accounts
+
+A message can reference a specific account as
+`[account name](credential://provider/credential_id)`. The account name is the
+user-facing label; the URI contains the exact provider and credential ID.
+Use that ID for the corresponding action instead of guessing an account by
+name or selecting a default. Separate references can name different accounts
+of the same provider in one message. These references do not grant access:
+normal user ownership and expert credential grants still apply. Never substitute
+a different account if the referenced account is unavailable. Show account
+names to the user, never credential IDs; preserve the reference when naming
+an account in your response so the UI can display its badge.
+
### Credentials & sign-in surfacing — CRITICAL
When the user asks for something that needs credentials (a block, an agent,
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/ChatInput.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/ChatInput.tsx
index 361b21c21a..d46dde2ac7 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/ChatInput.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/ChatInput.tsx
@@ -1,7 +1,8 @@
+import { CredentialMentionEditor } from "../CredentialMention/CredentialMentionEditor";
+import type { MentionInput } from "./useChatMentions";
import {
PromptInputButton,
PromptInputSubmit,
- PromptInputTextarea,
} from "@/components/ai-elements/prompt-input";
import { isGuidedPrompt } from "@/components/contextual/guidedPrompts";
import { toast } from "@/components/molecules/Toast/use-toast";
@@ -14,7 +15,6 @@ import {
import { cn } from "@/lib/utils";
import { Flag, useGetFlag } from "@/services/feature-flags/use-get-flag";
import {
- ChangeEvent,
ClipboardEvent,
KeyboardEvent,
ReactNode,
@@ -55,6 +55,7 @@ import {
} from "./helpers";
import { useChatInput } from "./useChatInput";
import { useChatMentions } from "./useChatMentions";
+import { useConnectedIntegrations } from "./useConnectedIntegrations";
import { useOnboardingMicGlow } from "./useOnboardingMicGlow";
import { useVoiceRecording } from "./useVoiceRecording";
import { ArrowUp02Icon } from "@hugeicons/core-free-icons";
@@ -176,12 +177,7 @@ export function ChatInput({
// during normal streaming (users can type and queue the next message).
const isTextareaDisabled = disabled || isUploadingFiles;
- const {
- value,
- setValue,
- handleSubmit,
- handleChange: baseHandleChange,
- } = useChatInput({
+ const { value, setValue, handleSubmit } = useChatInput({
onSend: async (message: string) => {
const { localFiles, workspaceAttachments } =
partitionAttachments(attachments);
@@ -207,14 +203,20 @@ export function ChatInput({
inputId,
});
+ const integrations = useConnectedIntegrations(expertId);
+
const mentions = useChatMentions({
- enabled: showWorkspaceFiles && !isBusy && !isAtCap,
+ enabled: !isBusy,
value,
setValue,
addWorkspaceFile: handleWorkspaceFileSelected,
addWorkspaceFolder: (folder, subfolderCount) =>
addAttachments([workspaceFolderToAttachment(folder, subfolderCount)]),
expertId,
+ // Files and folders become attachments, so the cap closes them off;
+ // integrations only edit the text and stay available.
+ includeWorkspaceFiles: showWorkspaceFiles && !isAtCap,
+ integrations,
});
const [isEnqueueing, setIsEnqueueing] = useState(false);
@@ -245,18 +247,18 @@ export function ChatInput({
isTranscribing,
});
- function handleChange(e: ChangeEvent<HTMLTextAreaElement>) {
+ function handleChange(nextValue: string, input: MentionInput) {
if (isRecording) return;
- baseHandleChange(e);
- mentions.detect(e.currentTarget);
+ setValue(nextValue);
+ mentions.detect(input);
}
- function handleKeyDown(e: KeyboardEvent<HTMLTextAreaElement>) {
+ function handleKeyDown(e: KeyboardEvent<HTMLElement>) {
if (mentions.onKeyDown(e)) return;
voiceHandleKeyDown(e);
}
- function handlePaste(e: ClipboardEvent<HTMLTextAreaElement>) {
+ function handlePaste(e: ClipboardEvent<HTMLElement>) {
if (isBusy) return;
const files = getFilesFromClipboard(e.clipboardData);
if (files.length === 0) return;
@@ -342,6 +344,8 @@ export function ChatInput({
{mentions.isOpen && (
<MentionDropdown
options={mentions.options}
+ showFiles={mentions.showFiles}
+ hasIntegrations={mentions.hasIntegrations}
isLoading={mentions.isLoading}
isError={mentions.isError}
highlightedIndex={mentions.highlightedIndex}
@@ -432,10 +436,10 @@ export function ChatInput({
stacked || isMultiline ? "order-first w-full" : "min-w-0 flex-1",
)}
>
- <PromptInputTextarea
+ <CredentialMentionEditor
id={inputId}
- aria-label="Chat message input"
value={value}
+ onInputReady={mentions.bindInput}
onChange={handleChange}
onKeyDown={handleKeyDown}
onPaste={handlePaste}
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/__tests__/integrationMentions.test.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/__tests__/integrationMentions.test.ts
new file mode 100644
index 0000000000..9f353653a3
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/__tests__/integrationMentions.test.ts
@@ -0,0 +1,173 @@
+import type { CredentialsMetaResponse } from "@/app/api/__generated__/models/credentialsMetaResponse";
+import { describe, expect, it } from "vitest";
+import {
+ connectedIntegrationsFromCredentials,
+ filterIntegrationMentions,
+ insertIntegrationMention,
+} from "../helpers";
+
+function credential(
+ provider: string,
+ overrides: Partial<CredentialsMetaResponse> = {},
+): CredentialsMetaResponse {
+ return {
+ id: `${provider}-${overrides.title ?? "cred"}`,
+ provider,
+ type: "api_key",
+ title: `${provider} account`,
+ scopes: null,
+ username: null,
+ ...overrides,
+ };
+}
+
+describe("connectedIntegrationsFromCredentials", () => {
+ it("keeps each account using its credential name", () => {
+ const result = connectedIntegrationsFromCredentials([
+ credential("google", {
+ title: "Work Gmail",
+ username: "[email protected]",
+ }),
+ credential("google", {
+ title: "Personal Gmail",
+ username: "[email protected]",
+ }),
+ ]);
+ expect(result).toEqual([
+ {
+ credentialId: "google-Personal Gmail",
+ provider: "google",
+ providerName: "Google",
+ name: "Personal Gmail",
+ username: "[email protected]",
+ token: "[Personal Gmail](credential://google/google-Personal%20Gmail)",
+ },
+ {
+ credentialId: "google-Work Gmail",
+ provider: "google",
+ providerName: "Google",
+ name: "Work Gmail",
+ username: "[email protected]",
+ token: "[Work Gmail](credential://google/google-Work%20Gmail)",
+ },
+ ]);
+ });
+
+ it("falls back to username, then provider when the title is blank", () => {
+ const result = connectedIntegrationsFromCredentials([
+ credential("google", { title: " ", username: "[email protected]" }),
+ credential("github", { title: null }),
+ ]);
+ expect(result.map((m) => m.name)).toEqual(["GitHub", "[email protected]"]);
+ });
+
+ it("disambiguates duplicate names independently of API ordering", () => {
+ const credentials = [
+ credential("google", { id: "account-1", title: "Work" }),
+ credential("google", { id: "account-2", title: "Work" }),
+ ];
+ const result = connectedIntegrationsFromCredentials(credentials);
+ expect(new Set(result.map((m) => m.token)).size).toBe(2);
+ expect(
+ connectedIntegrationsFromCredentials([...credentials].reverse()),
+ ).toEqual(result);
+ });
+
+ it("skips platform credit credentials, which are not something to address", () => {
+ const result = connectedIntegrationsFromCredentials([
+ credential("openai", { title: "Use Credits for OpenAI" }),
+ credential("anthropic", { title: "System key" }),
+ credential("slack"),
+ ]);
+ expect(result.map((mention) => mention.provider)).toEqual(["slack"]);
+ });
+
+ it("keeps Codex and OpenAI credentials separate with the OpenAI logo", () => {
+ const result = connectedIntegrationsFromCredentials([
+ credential("codex", { type: "oauth2", title: "Codex login" }),
+ credential("openai", { title: "API key" }),
+ ]);
+ expect(result.map((m) => [m.credentialId, m.provider, m.token])).toEqual([
+ [
+ "openai-API key",
+ "openai",
+ "[API key](credential://openai/openai-API%20key)",
+ ],
+ [
+ "codex-Codex login",
+ "codex",
+ "[Codex login](credential://codex/codex-Codex%20login)",
+ ],
+ ]);
+ });
+});
+
+describe("filterIntegrationMentions", () => {
+ const integrations = connectedIntegrationsFromCredentials([
+ credential("google_maps", { title: "Travel", username: "[email protected]" }),
+ credential("github", { title: "GitHub" }),
+ credential("hubspot", { title: "HubSpot" }),
+ ]);
+
+ it("returns everything for an empty query", () => {
+ expect(filterIntegrationMentions(integrations, "")).toEqual(integrations);
+ });
+
+ it("matches the display name case-insensitively and ignoring spaces", () => {
+ expect(
+ filterIntegrationMentions(integrations, "googlem").map((m) => m.name),
+ ).toEqual(["Travel"]);
+ expect(
+ filterIntegrationMentions(integrations, "HUB").map((m) => m.name),
+ ).toEqual(["GitHub", "HubSpot"]);
+ });
+
+ it("matches account names and usernames", () => {
+ expect(
+ filterIntegrationMentions(integrations, "TRAVEL").map((m) => m.name),
+ ).toEqual(["Travel"]);
+ expect(
+ filterIntegrationMentions(integrations, "me@example").map((m) => m.name),
+ ).toEqual(["Travel"]);
+ });
+
+ it("also matches the provider slug", () => {
+ expect(
+ filterIntegrationMentions(integrations, "_maps").map((m) => m.provider),
+ ).toEqual(["google_maps"]);
+ });
+});
+
+describe("insertIntegrationMention", () => {
+ const google = connectedIntegrationsFromCredentials([
+ credential("google", { title: "Work Gmail" }),
+ ])[0];
+
+ it("replaces the @query with the token and parks the caret after the existing space", () => {
+ expect(
+ insertIntegrationMention(
+ "check @goo for me",
+ { start: 6, end: 10 },
+ google,
+ ),
+ ).toEqual({
+ value: `check ${google.token} for me`,
+ caret: `check ${google.token} `.length,
+ });
+ });
+
+ it("adds a space when the mention ends the prompt", () => {
+ expect(
+ insertIntegrationMention("check @goo", { start: 6, end: 10 }, google),
+ ).toEqual({
+ value: `check ${google.token} `,
+ caret: `check ${google.token} `.length,
+ });
+ });
+
+ it("works when the mention is the whole prompt so far", () => {
+ expect(insertIntegrationMention("@", { start: 0, end: 1 }, google)).toEqual(
+ { value: `${google.token} `, caret: google.token.length + 1 },
+ );
+ });
+});
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/__tests__/useChatMentions.test.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/__tests__/useChatMentions.test.tsx
index 1119c25254..0469653afc 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/__tests__/useChatMentions.test.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/__tests__/useChatMentions.test.tsx
@@ -1,7 +1,15 @@
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
-import { act, renderHook, waitFor } from "@testing-library/react";
-import React, { type ReactNode } from "react";
+import {
+ act,
+ fireEvent,
+ render,
+ renderHook,
+ screen,
+ waitFor,
+} from "@testing-library/react";
+import React, { useState, type ReactNode } from "react";
import { afterEach, describe, expect, it, vi } from "vitest";
+import { connectedIntegrationsFromCredentials } from "../helpers";
import { useChatMentions } from "../useChatMentions";
const mockListWorkspaceFiles = vi.fn();
@@ -46,6 +54,23 @@ function keyEvent(key: string, composing: boolean | "keyCode229" = false) {
} as unknown as React.KeyboardEvent<HTMLTextAreaElement>;
}
+const GOOGLE = {
+ credentialId: "google-1",
+ providerName: "Google",
+ username: null,
+ provider: "google",
+ name: "Google",
+ token: "@Google",
+};
+const GITHUB = {
+ credentialId: "github-1",
+ providerName: "GitHub",
+ username: null,
+ provider: "github",
+ name: "GitHub",
+ token: "@GitHub",
+};
+
afterEach(() => {
vi.clearAllMocks();
});
@@ -289,4 +314,215 @@ describe("useChatMentions", () => {
expect(setValue).not.toHaveBeenCalled();
expect(addWorkspaceFile).not.toHaveBeenCalled();
});
+
+ it("lists matching integrations above the file results", async () => {
+ mockListWorkspaceFiles.mockResolvedValue({
+ status: 200,
+ data: { files: [FILE], has_more: false },
+ });
+
+ const { result } = renderHook(
+ () =>
+ useChatMentions({
+ enabled: true,
+ value: "hi @g",
+ setValue: vi.fn(),
+ addWorkspaceFile: vi.fn(),
+ addWorkspaceFolder: vi.fn(),
+ integrations: [GITHUB, GOOGLE],
+ }),
+ { wrapper: Wrapper },
+ );
+
+ act(() => result.current.detect(fakeTextarea("hi @g")));
+ expect(result.current.hasIntegrations).toBe(true);
+ expect(result.current.options).toEqual([
+ { kind: "integration", integration: GITHUB },
+ { kind: "integration", integration: GOOGLE },
+ ]);
+
+ await waitFor(() => expect(result.current.options).toHaveLength(3));
+ expect(result.current.options[2]).toEqual({ kind: "file", file: FILE });
+ });
+
+ it("narrows integrations by the typed query without waiting on the debounce", () => {
+ mockListWorkspaceFiles.mockResolvedValue({
+ status: 200,
+ data: { files: [], has_more: false },
+ });
+
+ const { result } = renderHook(
+ () =>
+ useChatMentions({
+ enabled: true,
+ value: "hi @goo",
+ setValue: vi.fn(),
+ addWorkspaceFile: vi.fn(),
+ addWorkspaceFolder: vi.fn(),
+ integrations: [GITHUB, GOOGLE],
+ }),
+ { wrapper: Wrapper },
+ );
+
+ act(() => result.current.detect(fakeTextarea("hi @goo")));
+ expect(result.current.options).toEqual([
+ { kind: "integration", integration: GOOGLE },
+ ]);
+ });
+
+ it("inserts the integration token into the prompt and moves the caret after it", async () => {
+ mockListWorkspaceFiles.mockResolvedValue({
+ status: 200,
+ data: { files: [], has_more: false },
+ });
+ const addWorkspaceFile = vi.fn();
+
+ function Harness() {
+ const [value, setValue] = useState("check ");
+ const mentions = useChatMentions({
+ enabled: true,
+ value,
+ setValue,
+ addWorkspaceFile,
+ addWorkspaceFolder: vi.fn(),
+ integrations: [GOOGLE],
+ });
+ return (
+ <textarea
+ aria-label="composer"
+ value={value}
+ onChange={(e) => {
+ setValue(e.target.value);
+ mentions.detect(e.currentTarget);
+ }}
+ onKeyDown={(e) => mentions.onKeyDown(e)}
+ />
+ );
+ }
+
+ render(<Harness />, { wrapper: Wrapper });
+ const textarea = screen.getByLabelText<HTMLTextAreaElement>("composer");
+
+ // Caret sits right after "@goo", with more text following it.
+ fireEvent.change(textarea, {
+ target: {
+ value: "check @goo for me",
+ selectionStart: "check @goo".length,
+ selectionEnd: "check @goo".length,
+ },
+ });
+ fireEvent.keyDown(textarea, { key: "Enter" });
+
+ expect(textarea.value).toBe("check @Google for me");
+ expect(addWorkspaceFile).not.toHaveBeenCalled();
+ // The browser would leave the caret at the end after the value swap; the
+ // hook parks it right after the inserted mention instead.
+ await waitFor(() =>
+ expect(textarea.selectionStart).toBe("check @Google ".length),
+ );
+ });
+
+ it("accepts two named accounts from one provider, including a query with spaces", () => {
+ const integrations = connectedIntegrationsFromCredentials([
+ {
+ id: "work",
+ provider: "google",
+ type: "oauth2",
+ title: "Work Gmail",
+ username: "[email protected]",
+ scopes: null,
+ },
+ {
+ id: "personal",
+ provider: "google",
+ type: "oauth2",
+ title: "Personal Gmail",
+ username: "[email protected]",
+ scopes: null,
+ },
+ ]);
+ function Harness() {
+ const [value, setValue] = useState("");
+ const mentions = useChatMentions({
+ enabled: true,
+ value,
+ setValue,
+ integrations,
+ includeWorkspaceFiles: false,
+ addWorkspaceFile: vi.fn(),
+ addWorkspaceFolder: vi.fn(),
+ });
+ return (
+ <textarea
+ aria-label="accounts"
+ value={value}
+ onChange={(e) => {
+ setValue(e.target.value);
+ mentions.detect(e.currentTarget);
+ }}
+ onKeyDown={(e) => mentions.onKeyDown(e)}
+ />
+ );
+ }
+ render(<Harness />, { wrapper: Wrapper });
+ const input = screen.getByLabelText<HTMLTextAreaElement>("accounts");
+ fireEvent.change(input, { target: { value: "Check my @Work G" } });
+ fireEvent.keyDown(input, { key: "Enter" });
+ expect(input.value).toBe(
+ "Check my [Work Gmail](credential://google/work) ",
+ );
+ fireEvent.change(input, {
+ target: { value: `${input.value}for new TODOs, and @Personal` },
+ });
+ fireEvent.keyDown(input, { key: "Enter" });
+ expect(input.value).toBe(
+ "Check my [Work Gmail](credential://google/work) for new TODOs, and [Personal Gmail](credential://google/personal) ",
+ );
+ });
+
+ it("never opens when workspace files are off and nothing is connected", () => {
+ const { result } = renderHook(
+ () =>
+ useChatMentions({
+ enabled: true,
+ value: "hi @",
+ setValue: vi.fn(),
+ addWorkspaceFile: vi.fn(),
+ addWorkspaceFolder: vi.fn(),
+ includeWorkspaceFiles: false,
+ integrations: [],
+ }),
+ { wrapper: Wrapper },
+ );
+
+ act(() => result.current.detect(fakeTextarea("hi @")));
+ expect(result.current.isOpen).toBe(false);
+ expect(result.current.options).toEqual([]);
+ expect(mockListWorkspaceFiles).not.toHaveBeenCalled();
+ });
+
+ it("offers only integrations and never queries files when workspace files are off", () => {
+ const { result } = renderHook(
+ () =>
+ useChatMentions({
+ enabled: true,
+ value: "hi @",
+ setValue: vi.fn(),
+ addWorkspaceFile: vi.fn(),
+ addWorkspaceFolder: vi.fn(),
+ includeWorkspaceFiles: false,
+ integrations: [GOOGLE],
+ }),
+ { wrapper: Wrapper },
+ );
+
+ act(() => result.current.detect(fakeTextarea("hi @")));
+ expect(result.current.isOpen).toBe(true);
+ expect(result.current.showFiles).toBe(false);
+ expect(result.current.isLoading).toBe(false);
+ expect(result.current.options).toEqual([
+ { kind: "integration", integration: GOOGLE },
+ ]);
+ expect(mockListWorkspaceFiles).not.toHaveBeenCalled();
+ });
});
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/__tests__/useConnectedIntegrations.test.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/__tests__/useConnectedIntegrations.test.tsx
new file mode 100644
index 0000000000..88168aed18
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/__tests__/useConnectedIntegrations.test.tsx
@@ -0,0 +1,119 @@
+import { getGetV1ListCredentialsQueryKey } from "@/app/api/__generated__/endpoints/integrations/integrations";
+import type { CredentialsMetaResponse } from "@/app/api/__generated__/models/credentialsMetaResponse";
+import { server } from "@/mocks/mock-server";
+import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
+import { renderHook, waitFor } from "@testing-library/react";
+import { http, HttpResponse } from "msw";
+import { type ReactNode } from "react";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { useConnectedIntegrations } from "../useConnectedIntegrations";
+
+const GOOGLE_CREDENTIAL: CredentialsMetaResponse = {
+ id: "google-cred",
+ provider: "google",
+ type: "oauth2",
+ title: "google account",
+ scopes: null,
+ username: null,
+};
+
+const listCredentials = vi.fn();
+
+function credentialsHandler() {
+ return http.get("*/api/integrations/credentials", () => {
+ listCredentials();
+ return HttpResponse.json([GOOGLE_CREDENTIAL]);
+ });
+}
+
+function createClient() {
+ return new QueryClient({
+ defaultOptions: { queries: { retry: false } },
+ });
+}
+
+function seedCredentials(client: QueryClient) {
+ client.setQueryData(getGetV1ListCredentialsQueryKey(), {
+ status: 200,
+ data: [GOOGLE_CREDENTIAL],
+ headers: new Headers(),
+ });
+}
+
+function wrapperFor(client: QueryClient) {
+ return function Wrapper({ children }: { children: ReactNode }) {
+ return (
+ <QueryClientProvider client={client}>{children}</QueryClientProvider>
+ );
+ };
+}
+
+afterEach(() => {
+ vi.clearAllMocks();
+ server.resetHandlers();
+});
+
+describe("useConnectedIntegrations", () => {
+ it("offers the owner's accounts in Otto", async () => {
+ server.use(credentialsHandler());
+ const { result } = renderHook(() => useConnectedIntegrations(), {
+ wrapper: wrapperFor(createClient()),
+ });
+ await waitFor(() =>
+ expect(result.current.map((account) => account.credentialId)).toEqual([
+ "google-cred",
+ ]),
+ );
+ });
+
+ it("only offers granted credentials for an expert even with owner credentials cached", async () => {
+ server.use(
+ credentialsHandler(),
+ http.get("*/api/experts/:expertId/credentials", ({ params }) =>
+ HttpResponse.json(
+ params.expertId === "expert-a"
+ ? [
+ {
+ credential_id: "work",
+ provider: "google",
+ title: "Work Gmail",
+ type: "oauth2",
+ },
+ ]
+ : [],
+ ),
+ ),
+ );
+ const client = createClient();
+ seedCredentials(client);
+ const { result, rerender } = renderHook(
+ ({ expertId }) => useConnectedIntegrations(expertId),
+ { wrapper: wrapperFor(client), initialProps: { expertId: "expert-a" } },
+ );
+ await waitFor(() =>
+ expect(result.current.map((account) => account.credentialId)).toEqual([
+ "work",
+ ]),
+ );
+ expect(listCredentials).not.toHaveBeenCalled();
+ rerender({ expertId: "expert-b" });
+ expect(result.current).toEqual([]);
+ await waitFor(() => expect(client.isFetching()).toBe(0));
+ expect(result.current).toEqual([]);
+ });
+
+ it("does not fall back to owner accounts when expert grants fail", async () => {
+ server.use(
+ http.get("*/api/experts/:expertId/credentials", () =>
+ HttpResponse.json({ detail: "unavailable" }, { status: 500 }),
+ ),
+ );
+ const client = createClient();
+ seedCredentials(client);
+ const { result } = renderHook(() => useConnectedIntegrations("expert-a"), {
+ wrapper: wrapperFor(client),
+ });
+ await waitFor(() => expect(client.isFetching()).toBe(0));
+ expect(result.current).toEqual([]);
+ });
+});
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/components/MentionDropdown.stories.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/components/MentionDropdown.stories.tsx
new file mode 100644
index 0000000000..1ca45ed833
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/components/MentionDropdown.stories.tsx
@@ -0,0 +1,166 @@
+import type { Meta, StoryObj } from "@storybook/nextjs";
+import { useState } from "react";
+import { expect, userEvent, within } from "storybook/test";
+import { PromptInputSubmit } from "@/components/ai-elements/prompt-input";
+import { InputGroup, InputGroupAddon } from "@/components/ui/input-group";
+import { CredentialMentionEditor } from "../../CredentialMention/CredentialMentionEditor";
+import { CredentialMentionMarkdown } from "../../CredentialMention/CredentialMentionMarkdown";
+import { CARD_SEND_BUTTON_CLASS } from "../helpers";
+import { useChatMentions } from "../useChatMentions";
+import { useConnectedIntegrations } from "../useConnectedIntegrations";
+import { MentionDropdown } from "./MentionDropdown";
+import { mentionStoryHandlers } from "./mentionStoryFixtures";
+
+interface Props {
+ expertId?: string;
+}
+
+function AccountMentionComposer({ expertId }: Props) {
+ const [value, setValue] = useState("");
+ const [sent, setSent] = useState("");
+ const [attachments, setAttachments] = useState<string[]>([]);
+ const integrations = useConnectedIntegrations(expertId);
+ const mentions = useChatMentions({
+ enabled: true,
+ value,
+ setValue,
+ integrations,
+ expertId,
+ includeWorkspaceFiles: true,
+ addWorkspaceFile: (file) =>
+ setAttachments((items) => [...items, file.name]),
+ addWorkspaceFolder: (folder) =>
+ setAttachments((items) => [...items, folder.name]),
+ });
+ return (
+ <div className="w-full max-w-2xl pt-80">
+ {sent && (
+ <div className="mb-8 flex justify-end">
+ <div
+ data-testid="sent-message"
+ className="max-w-full whitespace-pre-wrap rounded-3xl bg-zinc-100 px-4 py-2.5 text-base leading-8 text-zinc-900"
+ >
+ <CredentialMentionMarkdown>{sent}</CredentialMentionMarkdown>
+ </div>
+ </div>
+ )}
+ <form
+ className="relative"
+ onSubmit={(event) => {
+ event.preventDefault();
+ setSent(value);
+ setValue("");
+ }}
+ >
+ {mentions.isOpen && (
+ <MentionDropdown
+ {...mentions}
+ onSelect={mentions.accept}
+ onHighlight={mentions.setHighlightedIndex}
+ />
+ )}
+ <InputGroup className="flex-col gap-3 !rounded-3xl border-zinc-200 px-3.5 pb-3.5 pt-3 shadow-sm has-[[data-slot=input-group-control]:focus-visible]:ring-0">
+ {attachments.map((name) => (
+ <span
+ key={name}
+ className="rounded-lg bg-zinc-100 px-2 py-1 text-sm"
+ >
+ {name}
+ </span>
+ ))}
+ <CredentialMentionEditor
+ value={value}
+ onInputReady={mentions.bindInput}
+ onChange={(next, input) => {
+ setValue(next);
+ mentions.detect(input);
+ }}
+ onKeyDown={(event) => {
+ mentions.onKeyDown(event);
+ }}
+ onBlur={mentions.close}
+ placeholder="Type your message..."
+ className="w-full px-0.5 py-1"
+ />
+ <InputGroupAddon align="block-end" className="w-full justify-end p-0">
+ <PromptInputSubmit
+ disabled={!value.trim()}
+ className={CARD_SEND_BUTTON_CLASS}
+ />
+ </InputGroupAddon>
+ </InputGroup>
+ </form>
+ </div>
+ );
+}
+
+const meta = {
+ title: "Copilot/Account mentions",
+ component: AccountMentionComposer,
+ parameters: { layout: "fullscreen", msw: { handlers: mentionStoryHandlers } },
+ tags: ["autodocs"],
+} satisfies Meta<typeof AccountMentionComposer>;
+export default meta;
+type Story = StoryObj<typeof meta>;
+
+export const AccountPicker: Story = {
+ play: async ({ canvasElement }) => {
+ const canvas = within(canvasElement);
+ await userEvent.type(canvas.getByRole("textbox"), "Check my @");
+ await expect(
+ await canvas.findByRole("option", { name: /Work Gmail/ }),
+ ).toBeVisible();
+ await expect(
+ await canvas.findByRole("option", { name: /Team TODOs/ }),
+ ).toBeVisible();
+ },
+};
+
+export const ExpertAccounts: Story = {
+ args: { expertId: "expert-a" },
+ play: async ({ canvasElement }) => {
+ const canvas = within(canvasElement);
+ await userEvent.type(canvas.getByRole("textbox"), "Check my @");
+ await expect(
+ await canvas.findByRole("option", { name: /Work Gmail/ }),
+ ).toBeVisible();
+ await expect(
+ canvas.queryByRole("option", { name: /Personal Gmail/ }),
+ ).not.toBeInTheDocument();
+ },
+};
+
+async function fillAccountPrompt(canvasElement: HTMLElement) {
+ const canvas = within(canvasElement);
+ await userEvent.type(canvas.getByRole("textbox"), "Check my @Work");
+ await userEvent.click(
+ await canvas.findByRole("option", { name: /Work Gmail/ }),
+ );
+ await userEvent.type(
+ canvas.getByRole("textbox"),
+ "for new TODOs, and @Personal",
+ );
+ await userEvent.click(
+ await canvas.findByRole("option", { name: /Personal Gmail/ }),
+ );
+ await userEvent.type(canvas.getByRole("textbox"), "for work related emails.");
+ await expect(canvas.getByRole("textbox")).toHaveTextContent("Personal Gmail");
+}
+
+export const AccountsInInput: Story = {
+ play: async ({ canvasElement }) => {
+ await fillAccountPrompt(canvasElement);
+ },
+};
+
+export const SentMessage: Story = {
+ play: async ({ canvasElement }) => {
+ await fillAccountPrompt(canvasElement);
+ await userEvent.click(
+ within(canvasElement).getByRole("button", { name: "Submit" }),
+ );
+ await expect(
+ within(canvasElement).getByTestId("sent-message"),
+ ).toHaveTextContent("Personal Gmail");
+ },
+};
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/components/MentionDropdown.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/components/MentionDropdown.tsx
index 1be1cc1d13..dac2545f87 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/components/MentionDropdown.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/components/MentionDropdown.tsx
@@ -2,18 +2,29 @@
import { getFileTypeIcon } from "@/app/(platform)/artifacts/components/ArtifactsList/helpers";
import { folderSummary } from "@/app/(platform)/artifacts/components/WorkspaceFolders/folderTree";
+import type { WorkspaceFileItem } from "@/app/api/__generated__/models/workspaceFileItem";
+import type { WorkspaceFolder } from "@/app/api/__generated__/models/workspaceFolder";
+import { IntegrationLogo } from "@/components/molecules/IntegrationLogo/IntegrationLogo";
import { cn } from "@/lib/utils";
-import type { MutableRefObject } from "react";
+import type { MutableRefObject, ReactNode } from "react";
import {
AlertCircleIcon,
Folder01Icon,
Loading03Icon,
} from "@hugeicons/core-free-icons";
-import type { MentionOption } from "../useChatMentions";
import { Icon } from "@/components/atoms/Icon/Icon";
+import type { IntegrationMention } from "../helpers";
+import type { MentionOption } from "../useChatMentions";
interface Props {
+ /** Integrations first, then folders and files — indices match the
+ * highlight cursor. */
options: MentionOption[];
+ /** Whether workspace files are part of this picker; drives the file
+ * loading/error/empty copy. */
+ showFiles: boolean;
+ /** Whether the user has any integration to mention at all. */
+ hasIntegrations: boolean;
isLoading: boolean;
isError: boolean;
highlightedIndex: number;
@@ -24,6 +35,8 @@ interface Props {
export function MentionDropdown({
options,
+ showFiles,
+ hasIntegrations,
isLoading,
isError,
highlightedIndex,
@@ -32,17 +45,86 @@ export function MentionDropdown({
onHighlight,
}: Props) {
const showEmpty = !isLoading && !isError && options.length === 0;
+ const integrationCount = options.filter(
+ (option) => option.kind === "integration",
+ ).length;
+ const workspaceCount = options.length - integrationCount;
+ // Headings only earn their space when both kinds can appear together.
+ const showHeadings = showFiles && integrationCount > 0;
+ const showFilesHeading =
+ showHeadings && (workspaceCount > 0 || isLoading || isError);
+
+ function renderOption(
+ option: MentionOption,
+ index: number,
+ children: ReactNode,
+ ariaLabel?: string,
+ ) {
+ const isHighlighted = index === highlightedIndex;
+ return (
+ <button
+ key={mentionOptionKey(option)}
+ ref={isHighlighted ? highlightedRef : undefined}
+ type="button"
+ role="option"
+ aria-selected={isHighlighted}
+ aria-label={ariaLabel}
+ // preventDefault on mousedown keeps focus in the textarea so the
+ // caret/selection used to strip the @query stays valid.
+ onMouseDown={(e) => {
+ e.preventDefault();
+ onSelect(option);
+ }}
+ onMouseEnter={() => onHighlight(index)}
+ className={cn(
+ "flex w-full items-center gap-2 rounded-xl px-3 py-2 text-left text-sm",
+ isHighlighted ? "bg-zinc-100 text-zinc-900" : "text-zinc-700",
+ )}
+ >
+ {children}
+ </button>
+ );
+ }
return (
<div
role="listbox"
- aria-label="Workspace files"
+ aria-label="Mention suggestions"
// preventDefault on mousedown keeps focus in the textarea when clicking
// non-interactive areas (padding, empty/loading/error states) so the
// textarea's onBlur doesn't close the dropdown before a selection.
onMouseDown={(e) => e.preventDefault()}
- className="absolute bottom-full left-0 z-50 mb-2 max-h-60 w-72 overflow-y-auto rounded-2xl border border-zinc-200 bg-white p-1.5 shadow-md"
+ className="absolute bottom-full left-0 z-50 mb-2 max-h-80 w-80 max-w-full overflow-y-auto rounded-2xl border border-zinc-200 bg-white p-1.5 shadow-md"
>
+ {showHeadings && <SectionHeading label="Integrations" />}
+ {options.map((option, index) =>
+ option.kind === "integration"
+ ? renderOption(
+ option,
+ index,
+ <IntegrationOption integration={option.integration} />,
+ )
+ : null,
+ )}
+ {showFilesHeading && <SectionHeading label="Files" />}
+ {options.map((option, index) => {
+ if (option.kind === "folder") {
+ const count = folderSummary(
+ option.folder.file_count ?? 0,
+ option.subfolderCount,
+ );
+ return renderOption(
+ option,
+ index,
+ <FolderOption folder={option.folder} count={count} />,
+ `Folder ${option.folder.name}, ${count}`,
+ );
+ }
+ if (option.kind === "file") {
+ return renderOption(option, index, <FileOption file={option.file} />);
+ }
+ return null;
+ })}
{isError ? (
<p className="flex items-center gap-2 px-3 py-2 text-sm text-red-600">
<Icon icon={AlertCircleIcon} className="h-4 w-4 shrink-0" />
@@ -58,48 +140,91 @@ export function MentionDropdown({
</p>
) : null}
{showEmpty && (
- <p className="px-3 py-2 text-sm text-zinc-500">No matching files.</p>
+ <p className="px-3 py-2 text-sm text-zinc-500">
+ {emptyMessage(showFiles, hasIntegrations)}
+ </p>
)}
- {options.map((option, index) => {
- const isHighlighted = index === highlightedIndex;
- const isFolder = option.kind === "folder";
- const name = isFolder ? option.folder.name : option.file.name;
- const count = isFolder
- ? folderSummary(option.folder.file_count ?? 0, option.subfolderCount)
- : null;
- return (
- <button
- key={isFolder ? `folder-${option.folder.id}` : option.file.id}
- ref={isHighlighted ? highlightedRef : undefined}
- type="button"
- role="option"
- aria-selected={isHighlighted}
- aria-label={isFolder ? `Folder ${name}, ${count}` : undefined}
- // preventDefault on mousedown keeps focus in the textarea so the
- // caret/selection used to strip the @query stays valid.
- onMouseDown={(e) => {
- e.preventDefault();
- onSelect(option);
- }}
- onMouseEnter={() => onHighlight(index)}
- className={cn(
- "flex w-full items-center gap-2 rounded-xl px-3 py-2 text-left text-sm",
- isHighlighted ? "bg-zinc-100 text-zinc-900" : "text-zinc-700",
- )}
- >
- <Icon
- icon={
- isFolder ? Folder01Icon : getFileTypeIcon(option.file.mime_type)
- }
- className="h-4 w-4 shrink-0 text-zinc-900"
- />
- <span className="min-w-0 flex-1 truncate">{name}</span>
- {count ? (
- <span className="shrink-0 text-xs text-zinc-500">{count}</span>
- ) : null}
- </button>
- );
- })}
</div>
);
}
+
+function IntegrationOption({
+ integration,
+}: {
+ integration: IntegrationMention;
+}) {
+ return (
+ <>
+ <IntegrationLogo
+ provider={
+ integration.provider === "codex" ? "openai" : integration.provider
+ }
+ alt=""
+ size={16}
+ className="shrink-0"
+ />
+ <span className="min-w-0 flex-1">
+ <span className="block truncate font-medium" title={integration.name}>
+ {integration.name}
+ </span>
+ <span className="block truncate text-xs text-zinc-500">
+ {integration.providerName}
+ {integration.username ? ` · ${integration.username}` : ""}
+ </span>
+ </span>
+ </>
+ );
+}
+
+function FolderOption({
+ folder,
+ count,
+}: {
+ folder: WorkspaceFolder;
+ count: string;
+}) {
+ return (
+ <>
+ <Icon icon={Folder01Icon} className="h-4 w-4 shrink-0 text-zinc-900" />
+ <span className="min-w-0 flex-1 truncate">{folder.name}</span>
+ <span className="shrink-0 text-xs text-zinc-500">{count}</span>
+ </>
+ );
+}
+
+function FileOption({ file }: { file: WorkspaceFileItem }) {
+ return (
+ <>
+ <Icon
+ icon={getFileTypeIcon(file.mime_type)}
+ className="h-4 w-4 shrink-0 text-zinc-900"
+ />
+ <span className="min-w-0 flex-1 truncate">{file.name}</span>
+ </>
+ );
+}
+
+function SectionHeading({ label }: { label: string }) {
+ return (
+ <p
+ role="presentation"
+ className="px-3 pb-1 pt-2 text-left text-xs font-medium uppercase tracking-wide text-zinc-400"
+ >
+ {label}
+ </p>
+ );
+}
+
+function mentionOptionKey(option: MentionOption): string {
+ if (option.kind === "file") return `file:${option.file.id}`;
+ if (option.kind === "folder") return `folder:${option.folder.id}`;
+ return `integration:${option.integration.credentialId}`;
+}
+
+function emptyMessage(showFiles: boolean, hasIntegrations: boolean): string {
+ if (showFiles && hasIntegrations) return "No matching files or integrations.";
+ if (showFiles) return "No matching files.";
+ return hasIntegrations
+ ? "No matching integrations."
+ : "No connected integrations to mention.";
+}
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/components/__tests__/MentionDropdown.test.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/components/__tests__/MentionDropdown.test.tsx
index 8f033688d1..ae859c3fda 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/components/__tests__/MentionDropdown.test.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/components/__tests__/MentionDropdown.test.tsx
@@ -3,6 +3,7 @@ import type { MutableRefObject } from "react";
import { afterEach, describe, expect, it, vi } from "vitest";
import type { WorkspaceFileItem } from "@/app/api/__generated__/models/workspaceFileItem";
import type { WorkspaceFolder } from "@/app/api/__generated__/models/workspaceFolder";
+import { connectedIntegrationsFromCredentials } from "../../helpers";
import { MentionDropdown } from "../MentionDropdown";
import type { MentionOption } from "../../useChatMentions";
@@ -15,12 +16,26 @@ const FILE: WorkspaceFileItem = {
origin: "uploaded",
created_at: "2026-01-01T00:00:00Z",
};
+const FILE_ITEM: MentionOption = { kind: "file", file: FILE };
+const GOOGLE_ITEM: MentionOption = {
+ kind: "integration",
+ integration: {
+ credentialId: "google-1",
+ providerName: "Google",
+ username: null,
+ provider: "google",
+ name: "Google",
+ token: "@Google",
+ },
+};
function renderDropdown(
overrides: Partial<Parameters<typeof MentionDropdown>[0]> = {},
) {
const props = {
options: [] as MentionOption[],
+ showFiles: true,
+ hasIntegrations: false,
isLoading: false,
isError: false,
highlightedIndex: 0,
@@ -52,7 +67,19 @@ describe("MentionDropdown", () => {
it("shows an empty state when there are no matches", () => {
renderDropdown();
- expect(screen.getByText(/no matching files/i)).toBeTruthy();
+ expect(screen.getByText("No matching files.")).toBeTruthy();
+ });
+
+ it("names both kinds in the empty state when integrations are connected", () => {
+ renderDropdown({ hasIntegrations: true });
+ expect(screen.getByText("No matching files or integrations.")).toBeTruthy();
+ });
+
+ it("explains an integrations-only picker with nothing connected", () => {
+ renderDropdown({ showFiles: false });
+ expect(
+ screen.getByText("No connected integrations to mention."),
+ ).toBeTruthy();
});
it("renders a row per file and selects on mousedown", () => {
@@ -60,15 +87,79 @@ describe("MentionDropdown", () => {
const option = screen.getByRole("option", { name: /alpha\.txt/i });
fireEvent.mouseDown(option);
expect(onSelect).toHaveBeenCalledWith(fileOption(FILE));
+ expect(screen.queryByText("Files")).toBeNull();
+ });
+
+ it("renders integrations with their token under a heading above files", () => {
+ const { onSelect } = renderDropdown({
+ options: [GOOGLE_ITEM, FILE_ITEM],
+ hasIntegrations: true,
+ });
+ const options = screen.getAllByRole("option");
+ expect(options.map((option) => option.textContent)).toEqual([
+ "GoogleGoogle",
+ "alpha.txt",
+ ]);
+ expect(screen.getByText("Integrations")).toBeTruthy();
+ expect(screen.getByText("Files")).toBeTruthy();
+
+ fireEvent.mouseDown(options[0]);
+ expect(onSelect).toHaveBeenCalledWith(GOOGLE_ITEM);
});
- it("highlights a row on hover", () => {
+ it("shows separate accounts with their names and usernames and selects the chosen credential", () => {
+ const accounts = connectedIntegrationsFromCredentials([
+ {
+ id: "work",
+ provider: "google",
+ type: "oauth2",
+ title: "Work Gmail",
+ username: "[email protected]",
+ scopes: null,
+ },
+ {
+ id: "personal",
+ provider: "google",
+ type: "oauth2",
+ title: "Personal Gmail",
+ username: "[email protected]",
+ scopes: null,
+ },
+ ]);
+ const options: MentionOption[] = accounts.map((integration) => ({
+ kind: "integration",
+ integration,
+ }));
+ const { onSelect } = renderDropdown({ options, hasIntegrations: true });
+ expect(screen.getAllByRole("option")).toHaveLength(2);
+ const work = screen.getByRole("option", {
+ name: /Work Gmail.*[email protected]/,
+ });
+ fireEvent.mouseDown(work);
+ expect(onSelect).toHaveBeenCalledWith(options[1]);
+ expect(
+ screen.getByRole("option", { name: /Personal Gmail.*[email protected]/ }),
+ ).toBeTruthy();
+ });
+
+ it("drops the headings when files are not part of the picker", () => {
+ renderDropdown({
+ options: [GOOGLE_ITEM],
+ showFiles: false,
+ hasIntegrations: true,
+ });
+ expect(screen.getByRole("option", { name: /google/i })).toBeTruthy();
+ expect(screen.queryByText("Integrations")).toBeNull();
+ });
+
+ it("highlights a row on hover using its position in the combined list", () => {
const { onHighlight } = renderDropdown({
- options: [fileOption(FILE)],
+ options: [GOOGLE_ITEM, FILE_ITEM],
+ hasIntegrations: true,
highlightedIndex: -1,
});
fireEvent.mouseEnter(screen.getByRole("option", { name: /alpha\.txt/i }));
- expect(onHighlight).toHaveBeenCalledWith(0);
+ expect(onHighlight).toHaveBeenCalledWith(1);
});
it("offers folders above files, with their count in the accessible name", () => {
@@ -82,6 +173,21 @@ describe("MentionDropdown", () => {
expect(options[1].textContent).toContain("alpha.txt");
});
+ it("lists integrations, then folders, then files under the shared headings", () => {
+ renderDropdown({
+ options: [GOOGLE_ITEM, folderOption(FOLDER, 1), fileOption(FILE)],
+ hasIntegrations: true,
+ });
+ const options = screen.getAllByRole("option");
+ expect(options.map((option) => option.textContent)).toEqual([
+ "GoogleGoogle",
+ "Reports2 files",
+ "alpha.txt",
+ ]);
+ expect(screen.getByText("Integrations")).toBeTruthy();
+ expect(screen.getByText("Files")).toBeTruthy();
+ });
+
it("selects a folder option on mousedown", () => {
const { onSelect } = renderDropdown({
options: [folderOption(FOLDER, 0)],
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/components/mentionStoryFixtures.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/components/mentionStoryFixtures.ts
new file mode 100644
index 0000000000..eb4a44c73c
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/components/mentionStoryFixtures.ts
@@ -0,0 +1,79 @@
+import { http, HttpResponse } from "msw";
+import type { CredentialsMetaResponse } from "@/app/api/__generated__/models/credentialsMetaResponse";
+
+export const storyCredentials: CredentialsMetaResponse[] = [
+ {
+ id: "google-work",
+ provider: "google",
+ type: "oauth2",
+ title: "Work Gmail",
+ username: "[email protected]",
+ scopes: null,
+ },
+ {
+ id: "google-personal",
+ provider: "google",
+ type: "oauth2",
+ title: "Personal Gmail",
+ username: "[email protected]",
+ scopes: null,
+ },
+];
+
+export const mentionStoryHandlers = [
+ http.get("*/api/integrations/credentials", () =>
+ HttpResponse.json(storyCredentials),
+ ),
+ http.get("*/api/experts/:expertId/credentials", () =>
+ HttpResponse.json([
+ {
+ credential_id: "google-work",
+ provider: "google",
+ title: "Work Gmail",
+ type: "oauth2",
+ },
+ ]),
+ ),
+ http.get("*/api/workspace/files", ({ request }) => {
+ const query =
+ new URL(request.url).searchParams.get("q")?.toLowerCase() ?? "";
+ return HttpResponse.json({
+ files: [
+ {
+ id: "file-todos",
+ name: "Team TODOs.md",
+ path: "/workspace/Team TODOs.md",
+ mime_type: "text/markdown",
+ size_bytes: 1240,
+ origin: "uploaded",
+ created_at: "2026-09-24T10:00:00Z",
+ },
+ {
+ id: "file-notes",
+ name: "Meeting notes.pdf",
+ path: "/workspace/Meeting notes.pdf",
+ mime_type: "application/pdf",
+ size_bytes: 42000,
+ origin: "uploaded",
+ created_at: "2026-09-24T10:00:00Z",
+ },
+ ].filter((file) => file.name.toLowerCase().includes(query)),
+ has_more: false,
+ });
+ }),
+ http.get("*/api/workspace/folders", () =>
+ HttpResponse.json({
+ folders: [
+ {
+ id: "folder-project",
+ workspace_id: "workspace",
+ name: "Project files",
+ parent_id: null,
+ file_count: 4,
+ created_at: "2026-09-24T10:00:00Z",
+ updated_at: "2026-09-24T10:00:00Z",
+ },
+ ],
+ }),
+ ),
+];
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/helpers.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/helpers.ts
index c008ea0eda..ee07fa76ec 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/helpers.ts
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/helpers.ts
@@ -1,3 +1,8 @@
+import { serializeCredentialMention } from "../CredentialMention/helpers";
+import type { CredentialsMetaResponse } from "@/app/api/__generated__/models/credentialsMetaResponse";
+import { filterSystemCredentials } from "@/components/contextual/CredentialsInput/helpers";
+import { formatProviderName } from "@/components/contextual/IntegrationsPanel/helpers";
+
// Browsers name clipboard screenshots "image.png"; rename so multiple
// pasted images stay distinguishable in the composer and workspace.
const GENERIC_CLIPBOARD_IMAGE_NAME = /^image\.\w+$/i;
@@ -53,3 +58,81 @@ export function describeSendFailure(error: unknown, recovery: string) {
? `${reason} — ${recovery}.`
: `${recovery.charAt(0).toUpperCase()}${recovery.slice(1)}. Try again.`;
}
+
+export interface IntegrationMention {
+ credentialId: string;
+ provider: string;
+ providerName: string;
+ name: string;
+ username: string | null;
+ token: string;
+}
+
+export interface MentionRange {
+ start: number;
+ end: number;
+}
+
+export function connectedIntegrationsFromCredentials<
+ T extends Pick<
+ CredentialsMetaResponse,
+ "id" | "provider" | "title" | "username"
+ >,
+>(credentials: T[]): IntegrationMention[] {
+ const accounts = filterSystemCredentials(credentials)
+ .filter((credential) => credential.provider)
+ .map((credential) => {
+ const provider = credential.provider;
+ const providerName = formatProviderName(provider);
+ const name =
+ credential.title?.trim() || credential.username?.trim() || providerName;
+ const account = {
+ credentialId: credential.id,
+ provider,
+ providerName,
+ name,
+ username: credential.username,
+ };
+ return { ...account, token: serializeCredentialMention(account) };
+ });
+ return accounts.sort(
+ (a, b) =>
+ a.name.localeCompare(b.name) ||
+ a.credentialId.localeCompare(b.credentialId),
+ );
+}
+
+function normalizeMentionText(text: string): string {
+ return text.replace(/\s+/g, "").toLowerCase();
+}
+
+export function filterIntegrationMentions(
+ integrations: IntegrationMention[],
+ query: string,
+): IntegrationMention[] {
+ const q = normalizeMentionText(query);
+ if (!q) return integrations;
+ return integrations.filter(
+ (integration) =>
+ normalizeMentionText(integration.name).includes(q) ||
+ normalizeMentionText(integration.provider).includes(q) ||
+ normalizeMentionText(integration.providerName).includes(q) ||
+ normalizeMentionText(integration.username ?? "").includes(q),
+ );
+}
+
+/** Replaces the `@query` under the caret with the integration's token and
+ * makes sure a space follows it, so typing continues after the mention.
+ * Returns the new text and where the caret belongs. */
+export function insertIntegrationMention(
+ value: string,
+ range: MentionRange,
+ integration: IntegrationMention,
+): { value: string; caret: number } {
+ const rest = value.slice(range.end);
+ const separator = /^\s/.test(rest) ? "" : " ";
+ return {
+ value: value.slice(0, range.start) + integration.token + separator + rest,
+ caret: range.start + integration.token.length + 1,
+ };
+}
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/useChatMentions.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/useChatMentions.ts
index 23a5ab895d..0c648ec37f 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/useChatMentions.ts
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/useChatMentions.ts
@@ -10,13 +10,28 @@ import { useDebouncedValue } from "@/hooks/useDebouncedValue";
import { useKeyboardNav } from "@/components/organisms/SearchCommandModal/useKeyboardNav";
import { useQuery } from "@tanstack/react-query";
import type { KeyboardEvent } from "react";
-import { useState } from "react";
+import { useEffect, useRef, useState } from "react";
import { isKey } from "@/lib/keyboard";
+import {
+ filterIntegrationMentions,
+ insertIntegrationMention,
+ type IntegrationMention,
+} from "./helpers";
const MENTION_RE = /(?:^|\s)@([^\s@]*)$/;
+// Account names can contain spaces ("Work Gmail"), so this one keeps matching
+// past whitespace; it only wins when the wider token still names an account.
+const ACCOUNT_MENTION_RE = /(?:^|\s)@([^@\n]*)$/;
const QUERY_DEBOUNCE_MS = 200;
const MENTION_RESULT_LIMIT = 8;
const MENTION_FOLDER_LIMIT = 3;
+const INTEGRATION_RESULT_LIMIT = 6;
+
+export interface MentionInput {
+ value: string;
+ selectionStart: number | null;
+ setSelectionRange: (start: number, end: number) => void;
+}
interface ActiveMention {
query: string;
@@ -24,6 +39,11 @@ interface ActiveMention {
end: number;
}
+export type MentionOption =
+ | { kind: "file"; file: WorkspaceFileItem }
+ | { kind: "folder"; folder: WorkspaceFolder; subfolderCount: number }
+ | { kind: "integration"; integration: IntegrationMention };
+
interface Args {
enabled: boolean;
value: string;
@@ -32,13 +52,20 @@ interface Args {
addWorkspaceFolder: (folder: WorkspaceFolder, subfolderCount: number) => void;
/** Expert the chat is scoped to; suggests only files that expert can attach. */
expertId?: string | null;
+ /** False while the workspace-files flag is off: the picker then only
+ * offers integrations and never queries the file or folder APIs. */
+ includeWorkspaceFiles?: boolean;
+ /** Connected integrations offered above the folder and file results. */
+ integrations?: IntegrationMention[];
}
/**
- * Detects an active `@token` at the textarea caret and drives a workspace-file
- * autocomplete. Selecting a file strips the `@query` from the message and adds
- * the file as an attachment chip. Keyboard nav stays in the textarea (focus
- * never leaves), so this owns the highlight cursor and the key handler.
+ * Detects an active `@token` at the textarea caret and drives a mention
+ * autocomplete over connected integrations, workspace folders and workspace
+ * files. Selecting a file or folder strips the `@query` from the message and
+ * adds it as an attachment chip; selecting an integration replaces the
+ * `@query` with a credential reference rendered as an inline badge. Keyboard nav stays in the textarea (focus never leaves), so
+ * this owns the highlight cursor and the key handler.
*/
export function useChatMentions({
enabled,
@@ -47,14 +74,16 @@ export function useChatMentions({
addWorkspaceFile,
addWorkspaceFolder,
expertId,
+ includeWorkspaceFiles = true,
+ integrations = [],
}: Args) {
const [active, setActive] = useState<ActiveMention | null>(null);
const isOpen = enabled && active !== null;
+ const textareaRef = useRef<MentionInput | null>(null);
+ const [pendingCaret, setPendingCaret] = useState<number | null>(null);
- const debouncedQuery = useDebouncedValue(
- active?.query ?? "",
- QUERY_DEBOUNCE_MS,
- );
+ const query = active?.query ?? "";
+ const debouncedQuery = useDebouncedValue(query, QUERY_DEBOUNCE_MS);
const search = useQuery({
queryKey: [
@@ -72,7 +101,7 @@ export function useChatMentions({
// the chat may attach — the picker's narrower default is for browsing.
include_user_files: expertId ? true : undefined,
}),
- enabled: isOpen,
+ enabled: isOpen && includeWorkspaceFiles,
// Keep results while the same expert's query refines; drop them when the
// chat switches expert so no foreign file can be picked mid-request.
placeholderData: (previousData, previousQuery) =>
@@ -82,21 +111,37 @@ export function useChatMentions({
});
const files =
- search.data?.status === 200 ? (search.data.data.files ?? []) : [];
+ includeWorkspaceFiles && search.data?.status === 200
+ ? (search.data.data.files ?? [])
+ : [];
const foldersQuery = useListWorkspaceFolders({
- query: { select: okData, enabled: isOpen },
+ query: { select: okData, enabled: isOpen && includeWorkspaceFiles },
});
- const allFolders = foldersQuery.data?.folders ?? [];
+ const allFolders = includeWorkspaceFiles
+ ? (foldersQuery.data?.folders ?? [])
+ : [];
const folders = matchFolders(allFolders, debouncedQuery);
- // Folders come first, so the keyboard cursor spans both groups as one list.
+
+ const matchedIntegrations = filterIntegrationMentions(
+ integrations,
+ query,
+ ).slice(0, INTEGRATION_RESULT_LIMIT);
+
+ // Integrations, then folders, then files: the keyboard cursor spans all
+ // three groups as one list.
const options: MentionOption[] = [
- ...folders.map((folder) => ({
- kind: "folder" as const,
- folder,
- subfolderCount: subfolderCountOf(allFolders, folder.id),
- })),
- ...files.map((file) => ({ kind: "file" as const, file })),
+ ...matchedIntegrations.map(
+ (integration): MentionOption => ({ kind: "integration", integration }),
+ ),
+ ...folders.map(
+ (folder): MentionOption => ({
+ kind: "folder",
+ folder,
+ subfolderCount: subfolderCountOf(allFolders, folder.id),
+ }),
+ ),
+ ...files.map((file): MentionOption => ({ kind: "file", file })),
];
const {
@@ -104,18 +149,48 @@ export function useChatMentions({
highlightedRef,
moveHighlight,
setHighlightedIndex,
- } = useKeyboardNav(options.length, debouncedQuery);
+ } = useKeyboardNav(options.length, query);
- function detect(textarea: HTMLTextAreaElement) {
+ // The caret lands wherever the browser puts it after a programmatic value
+ // change (usually the end); move it to just after the inserted mention once
+ // the new text has been committed to the textarea.
+ useEffect(() => {
+ if (pendingCaret === null) return;
+ const textarea = textareaRef.current;
+ if (textarea && textarea.value.length >= pendingCaret) {
+ textarea.setSelectionRange(pendingCaret, pendingCaret);
+ }
+ setPendingCaret(null);
+ }, [pendingCaret, value]);
+
+ // With workspace files off and nothing connected there is nothing to pick,
+ // so a typed `@` must not open an empty dropdown.
+ const hasOptions = includeWorkspaceFiles || integrations.length > 0;
+
+ function detect(textarea: MentionInput) {
if (!enabled) return;
+ textareaRef.current = textarea;
+ if (!hasOptions) {
+ setActive(null);
+ return;
+ }
const caret = textarea.selectionStart ?? textarea.value.length;
- const match = textarea.value.slice(0, caret).match(MENTION_RE);
+ const beforeCaret = textarea.value.slice(0, caret);
+ // A single-word token is always a query; a multi-word one is only kept
+ // while it still matches a connected account.
+ const accountMatch = beforeCaret.match(ACCOUNT_MENTION_RE);
+ const match =
+ beforeCaret.match(MENTION_RE) ??
+ (accountMatch &&
+ filterIntegrationMentions(integrations, accountMatch[1]).length > 0
+ ? accountMatch
+ : null);
if (!match) {
setActive(null);
return;
}
- const query = match[1];
- setActive({ query, start: caret - query.length - 1, end: caret });
+ const token = match[1];
+ setActive({ query: token, start: caret - token.length - 1, end: caret });
}
function close() {
@@ -127,14 +202,20 @@ export function useChatMentions({
// list can momentarily leave it pointing past the end — guard against the
// out-of-bounds `undefined` before touching the option.
if (!active || !option) return;
- setValue(value.slice(0, active.start) + value.slice(active.end));
- if (option.kind === "folder")
- addWorkspaceFolder(option.folder, option.subfolderCount);
- else addWorkspaceFile(option.file);
+ if (option.kind === "integration") {
+ const next = insertIntegrationMention(value, active, option.integration);
+ setValue(next.value);
+ setPendingCaret(next.caret);
+ } else {
+ setValue(value.slice(0, active.start) + value.slice(active.end));
+ if (option.kind === "folder")
+ addWorkspaceFolder(option.folder, option.subfolderCount);
+ else addWorkspaceFile(option.file);
+ }
setActive(null);
}
- function onKeyDown(e: KeyboardEvent<HTMLTextAreaElement>): boolean {
+ function onKeyDown(e: KeyboardEvent<HTMLElement>): boolean {
if (!isOpen) return false;
if (isKey(e, "Escape")) {
e.preventDefault();
@@ -163,11 +244,16 @@ export function useChatMentions({
return {
isOpen,
options,
- isLoading: search.isLoading,
- isError: search.isError,
+ showFiles: includeWorkspaceFiles,
+ hasIntegrations: integrations.length > 0,
+ isLoading: includeWorkspaceFiles && search.isLoading,
+ isError: includeWorkspaceFiles && search.isError,
highlightedIndex,
highlightedRef,
setHighlightedIndex,
+ bindInput: (input: MentionInput) => {
+ textareaRef.current = input;
+ },
detect,
close,
accept,
@@ -175,10 +261,6 @@ export function useChatMentions({
};
}
-export type MentionOption =
- | { kind: "file"; file: WorkspaceFileItem }
- | { kind: "folder"; folder: WorkspaceFolder; subfolderCount: number };
-
function matchFolders(
folders: WorkspaceFolder[],
query: string,
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/useConnectedIntegrations.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/useConnectedIntegrations.ts
new file mode 100644
index 0000000000..01e3c178da
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/useConnectedIntegrations.ts
@@ -0,0 +1,32 @@
+import { useGetV1ListCredentials } from "@/app/api/__generated__/endpoints/integrations/integrations";
+import { useListExpertCredentials } from "@/app/api/__generated__/endpoints/experts/experts";
+import { connectedIntegrationsFromCredentials } from "./helpers";
+
+export function useConnectedIntegrations(expertId?: string | null) {
+ const credentials = useGetV1ListCredentials({
+ query: {
+ enabled: !expertId,
+ select: (response) => (response.status === 200 ? response.data : []),
+ },
+ });
+ const grants = useListExpertCredentials(expertId ?? "", {
+ query: {
+ enabled: Boolean(expertId),
+ select: (response) => (response.status === 200 ? response.data : []),
+ },
+ });
+ if (expertId) {
+ if (grants.isError) return [];
+ return connectedIntegrationsFromCredentials(
+ (grants.data ?? []).map((credential) => ({
+ id: credential.credential_id,
+ provider: credential.provider,
+ title: credential.title,
+ username: null,
+ })),
+ );
+ }
+ return connectedIntegrationsFromCredentials(
+ credentials.isError ? [] : (credentials.data ?? []),
+ );
+}
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/useVoiceRecording.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/useVoiceRecording.ts
index 20c903095d..5ab3d7b823 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/useVoiceRecording.ts
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatInput/useVoiceRecording.ts
@@ -279,7 +279,7 @@ export function useVoiceRecording({
}, [isTranscribing, inputId]);
const handleKeyDown = useCallback(
- (event: KeyboardEvent<HTMLTextAreaElement>) => {
+ (event: KeyboardEvent<HTMLElement>) => {
// Allow space to toggle recording (start when empty, stop when recording)
if (isKey(event, " ") && !isTranscribing) {
if (isRecordingRef.current) {
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/ChatMessagesContainer.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/ChatMessagesContainer.tsx
index f5fe89ad4e..01bb82f057 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/ChatMessagesContainer.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/ChatMessagesContainer.tsx
@@ -1,3 +1,4 @@
+import { CredentialMentionText } from "../CredentialMention/CredentialMentionText";
import { cn } from "@/lib/utils";
import { useMemo, useState } from "react";
import {
@@ -860,7 +861,9 @@ export function ChatMessagesContainer({
: "rounded-3xl text-[1rem] leading-relaxed",
)}
>
- <span>{msg}</span>
+ <span>
+ <CredentialMentionText text={msg} />
+ </span>
<span className="flex items-center gap-1 text-xs text-slate-500">
<Icon icon={Clock01Icon} className="size-3" />
Queued
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/__tests__/ChatMessagesContainer.test.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/__tests__/ChatMessagesContainer.test.tsx
index 68961838b1..065848ecb2 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/__tests__/ChatMessagesContainer.test.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/__tests__/ChatMessagesContainer.test.tsx
@@ -271,6 +271,20 @@ describe("ChatMessagesContainer — queuedMessages", () => {
expect(screen.getByText("Queued")).toBeDefined();
});
+ it("renders queued account references as badges without exposing IDs", () => {
+ const { container } = render(
+ <ChatMessagesContainer
+ {...baseProps}
+ queuedMessages={[
+ "Check [Work Gmail](credential://google/work-credential-id)",
+ ]}
+ />,
+ );
+ expect(screen.getByText("Work Gmail")).toBeDefined();
+ expect(container.textContent).not.toContain("work-credential-id");
+ expect(container.textContent).not.toContain("credential://");
+ });
+
it("renders multiple queued messages as separate bubbles", () => {
render(
<ChatMessagesContainer
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/CopyButton.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/CopyButton.tsx
index 3389d7ec8d..b1ffceb802 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/CopyButton.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/CopyButton.tsx
@@ -1,5 +1,6 @@
"use client";
+import { credentialMentionDisplayText } from "../../CredentialMention/helpers";
import { MessageAction } from "@/components/ai-elements/message";
import { toast } from "@/components/molecules/Toast/use-toast";
import { useState } from "react";
@@ -17,7 +18,7 @@ export function CopyButton({ text }: Props) {
async function handleCopy() {
try {
- await navigator.clipboard.writeText(text);
+ await navigator.clipboard.writeText(credentialMentionDisplayText(text));
setCopied(true);
setTimeout(() => setCopied(false), 2000);
} catch {
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/MessagePartRenderer.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/MessagePartRenderer.tsx
index 96187e28f6..5d286d87f3 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/MessagePartRenderer.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/MessagePartRenderer.tsx
@@ -1,4 +1,4 @@
-import { MessageResponse } from "@/components/ai-elements/message";
+import { CredentialMentionMarkdown } from "../../CredentialMention/CredentialMentionMarkdown";
import { ErrorCard } from "@/components/molecules/ErrorCard/ErrorCard";
import { StoppedTaskCard } from "./StoppedTaskCard";
import { ToolUIPart, UIDataTypes, UIMessage, UITools } from "ai";
@@ -95,12 +95,12 @@ function TextWithArtifactCards({
// Text reads first, with the artifact cards trailing.
return (
<>
- <MessageResponse
+ <CredentialMentionMarkdown
components={STREAMDOWN_COMPONENTS}
className="[&_li]:py-0"
>
{resolved}
- </MessageResponse>
+ </CredentialMentionMarkdown>
{artifacts.length > 0 && (
<div className="mt-2 grid grid-cols-1 gap-1 sm:grid-cols-2">
{artifacts.map((artifact) => (
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/PendingUploadMessage.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/PendingUploadMessage.tsx
index 9eaa751b7a..b15c9b5681 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/PendingUploadMessage.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/PendingUploadMessage.tsx
@@ -1,5 +1,6 @@
"use client";
+import { CredentialMentionText } from "../../CredentialMention/CredentialMentionText";
import { Icon } from "@/components/atoms/Icon/Icon";
import { Message, MessageContent } from "@/components/ai-elements/message";
import { cn } from "@/lib/utils";
@@ -101,7 +102,7 @@ export function PendingUploadMessage({ pendingSend, isCompact }: Props) {
)}
>
<div className="whitespace-pre-wrap break-words">
- {pendingSend.text}
+ <CredentialMentionText text={pendingSend.text} />
</div>
</MessageContent>
)}
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/CredentialMention/CredentialMentionBadge.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/CredentialMention/CredentialMentionBadge.tsx
new file mode 100644
index 0000000000..697fe31511
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/CredentialMention/CredentialMentionBadge.tsx
@@ -0,0 +1,20 @@
+import { IntegrationLogo } from "@/components/molecules/IntegrationLogo/IntegrationLogo";
+
+interface Props {
+ name: string;
+ provider: string;
+}
+
+export function CredentialMentionBadge({ name, provider }: Props) {
+ return (
+ <span className="mx-0.5 inline-flex max-w-full items-center gap-1.5 rounded-xl border border-blue-200 bg-blue-50 px-1.5 py-0.5 align-baseline text-sm font-medium leading-5 text-blue-900">
+ <IntegrationLogo
+ provider={provider === "codex" ? "openai" : provider}
+ alt=""
+ size={14}
+ className="shrink-0"
+ />
+ <span className="truncate">{name}</span>
+ </span>
+ );
+}
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/CredentialMention/CredentialMentionEditor.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/CredentialMention/CredentialMentionEditor.tsx
new file mode 100644
index 0000000000..cbd48741c4
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/CredentialMention/CredentialMentionEditor.tsx
@@ -0,0 +1,189 @@
+import { createPortal } from "react-dom";
+import {
+ useId,
+ useLayoutEffect,
+ useRef,
+ type ChangeEvent,
+ type ClipboardEvent,
+ type KeyboardEvent,
+} from "react";
+import { PromptInputTextarea } from "@/components/ai-elements/prompt-input";
+import { parseCredentialMentions } from "./helpers";
+import { isKey } from "@/lib/keyboard";
+import { cn } from "@/lib/utils";
+import type { MentionInput } from "../ChatInput/useChatMentions";
+import { CredentialMentionBadge } from "./CredentialMentionBadge";
+import {
+ editorSelectionRange,
+ mentionInputFor,
+ readMentionEditor,
+ setEditorCaret,
+} from "./editorHelpers";
+import { useCredentialMentionEditor } from "./useCredentialMentionEditor";
+
+interface Props {
+ id?: string;
+ value: string;
+ onChange: (value: string, input: MentionInput) => void;
+ onKeyDown: (event: KeyboardEvent<HTMLElement>) => void;
+ onPaste?: (event: ClipboardEvent<HTMLElement>) => void;
+ onBlur?: () => void;
+ onInputReady?: (input: MentionInput) => void;
+ onMultilineChange?: (multiline: boolean) => void;
+ disabled?: boolean;
+ placeholder?: string;
+ className?: string;
+}
+
+/** Where the caret sat right after the edit that produced `value`, so it can
+ * be put back when that edit swaps the textarea for the badge editor or the
+ * other way round. */
+interface PendingSelection {
+ value: string;
+ start: number;
+ end: number;
+}
+
+function restoreSelection(input: HTMLElement, selection: PendingSelection) {
+ if (input instanceof HTMLTextAreaElement)
+ input.setSelectionRange(selection.start, selection.end);
+ else setEditorCaret(input, selection.start, selection.end);
+}
+
+export function CredentialMentionEditor({
+ id,
+ value,
+ onChange,
+ onKeyDown,
+ onPaste,
+ onBlur,
+ onInputReady,
+ onMultilineChange,
+ disabled,
+ placeholder,
+ className,
+}: Props) {
+ const fallbackId = useId();
+ const inputId = id ?? fallbackId;
+ const rich = parseCredentialMentions(value).some(
+ (part) => typeof part !== "string",
+ );
+ const wasRich = useRef(rich);
+ const pendingSelection = useRef<PendingSelection | null>(null);
+ const {
+ editorRef,
+ attachEditor,
+ badges,
+ resync,
+ onCompositionStart,
+ onCompositionEnd,
+ } = useCredentialMentionEditor(value, onMultilineChange);
+
+ useLayoutEffect(() => {
+ const input = document.getElementById(inputId);
+ const pending = pendingSelection.current;
+ pendingSelection.current = null;
+ if (input && wasRich.current !== rich) {
+ input.focus();
+ if (pending?.value === value) restoreSelection(input, pending);
+ }
+ wasRich.current = rich;
+ if (rich && editorRef.current)
+ onInputReady?.(mentionInputFor(editorRef.current));
+ });
+
+ function handleInput() {
+ const editor = editorRef.current;
+ if (!editor) return;
+ const next = readMentionEditor(editor);
+ const selection = editorSelectionRange(editor);
+ pendingSelection.current = selection && { value: next, ...selection };
+ onChange(next, mentionInputFor(editor));
+ resync();
+ }
+
+ function handleTextareaChange(event: ChangeEvent<HTMLTextAreaElement>) {
+ const { value: next, selectionStart, selectionEnd } = event.currentTarget;
+ pendingSelection.current = {
+ value: next,
+ start: selectionStart,
+ end: selectionEnd,
+ };
+ onChange(next, event.currentTarget);
+ }
+
+ function handleKeyDown(event: KeyboardEvent<HTMLDivElement>) {
+ onKeyDown(event);
+ if (event.defaultPrevented || !isKey(event, "Enter")) return;
+ event.preventDefault();
+ if (event.shiftKey) document.execCommand("insertText", false, "\n");
+ else event.currentTarget.closest("form")?.requestSubmit();
+ }
+
+ function handlePaste(event: ClipboardEvent<HTMLDivElement>) {
+ onPaste?.(event);
+ if (event.defaultPrevented) return;
+ event.preventDefault();
+ document.execCommand(
+ "insertText",
+ false,
+ event.clipboardData.getData("text/plain"),
+ );
+ }
+
+ if (!rich)
+ return (
+ <PromptInputTextarea
+ id={inputId}
+ aria-label="Chat message input"
+ value={value}
+ onChange={handleTextareaChange}
+ onKeyDown={onKeyDown}
+ onPaste={onPaste}
+ onBlur={onBlur}
+ onMultilineChange={onMultilineChange}
+ disabled={disabled}
+ placeholder={placeholder}
+ className={cn("text-left", className)}
+ />
+ );
+
+ return (
+ <>
+ <input type="hidden" name="message" value={value} />
+ <div
+ ref={attachEditor}
+ id={inputId}
+ role="textbox"
+ aria-label="Chat message input"
+ aria-multiline="true"
+ aria-disabled={disabled}
+ data-slot="input-group-control"
+ data-placeholder={placeholder}
+ contentEditable={!disabled}
+ suppressContentEditableWarning
+ onInput={handleInput}
+ onKeyDown={handleKeyDown}
+ onPaste={handlePaste}
+ onCompositionStart={onCompositionStart}
+ onCompositionEnd={onCompositionEnd}
+ onBlur={onBlur}
+ className={cn(
+ "max-h-48 min-h-10 w-full overflow-y-auto whitespace-pre-wrap break-words bg-transparent px-3 py-2 text-left text-base leading-7 outline-none empty:before:pointer-events-none empty:before:text-zinc-400 empty:before:content-[attr(data-placeholder)]",
+ disabled && "cursor-not-allowed opacity-50",
+ className,
+ )}
+ />
+ {badges.map(({ element, mention }, index) =>
+ createPortal(
+ <CredentialMentionBadge
+ name={mention.name}
+ provider={mention.provider}
+ />,
+ element,
+ `${mention.credentialId}-${index}`,
+ ),
+ )}
+ </>
+ );
+}
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/CredentialMention/CredentialMentionMarkdown.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/CredentialMention/CredentialMentionMarkdown.tsx
new file mode 100644
index 0000000000..6609907e7c
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/CredentialMention/CredentialMentionMarkdown.tsx
@@ -0,0 +1,42 @@
+import { createElement } from "react";
+import {
+ MessageResponse,
+ type MessageResponseProps,
+} from "@/components/ai-elements/message";
+import { CredentialMentionBadge } from "./CredentialMentionBadge";
+import {
+ credentialMentionImageProvider,
+ credentialMentionsToMarkdown,
+} from "./helpers";
+
+interface Props extends Omit<MessageResponseProps, "children"> {
+ children: string;
+}
+
+export function CredentialMentionMarkdown({
+ children,
+ components,
+ ...props
+}: Props) {
+ const Image = components?.img;
+ function renderImage(imageProps: React.JSX.IntrinsicElements["img"]) {
+ const provider = credentialMentionImageProvider(imageProps.src);
+ if (provider !== null)
+ return (
+ <CredentialMentionBadge
+ provider={provider}
+ name={imageProps.alt ?? "Account"}
+ />
+ );
+ if (typeof Image === "string") return createElement(Image, imageProps);
+ return Image ? <Image {...imageProps} /> : null;
+ }
+ return (
+ <MessageResponse
+ {...props}
+ components={{ ...components, img: renderImage }}
+ >
+ {credentialMentionsToMarkdown(children)}
+ </MessageResponse>
+ );
+}
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/CredentialMention/CredentialMentionText.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/CredentialMention/CredentialMentionText.tsx
new file mode 100644
index 0000000000..c56a317b72
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/CredentialMention/CredentialMentionText.tsx
@@ -0,0 +1,23 @@
+import { Fragment } from "react";
+import { CredentialMentionBadge } from "./CredentialMentionBadge";
+import { parseCredentialMentions } from "./helpers";
+
+interface Props {
+ text: string;
+}
+
+export function CredentialMentionText({ text }: Props) {
+ return (
+ <>
+ {parseCredentialMentions(text).map((part, index) => (
+ <Fragment key={index}>
+ {typeof part === "string" ? (
+ part
+ ) : (
+ <CredentialMentionBadge name={part.name} provider={part.provider} />
+ )}
+ </Fragment>
+ ))}
+ </>
+ );
+}
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/CredentialMention/__tests__/CredentialMentionEditor.test.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/CredentialMention/__tests__/CredentialMentionEditor.test.tsx
new file mode 100644
index 0000000000..3ed02fc0f1
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/CredentialMention/__tests__/CredentialMentionEditor.test.tsx
@@ -0,0 +1,270 @@
+import {
+ fireEvent,
+ render,
+ screen,
+ waitFor,
+} from "@/tests/integrations/test-utils";
+import userEvent from "@testing-library/user-event";
+import { useState } from "react";
+import { describe, expect, it, vi } from "vitest";
+import { CredentialMentionEditor } from "../CredentialMentionEditor";
+import { useChatMentions } from "../../ChatInput/useChatMentions";
+import { connectedIntegrationsFromCredentials } from "../../ChatInput/helpers";
+
+const accounts = connectedIntegrationsFromCredentials([
+ {
+ id: "work-credential-id",
+ provider: "google",
+ type: "oauth2",
+ title: "Work Gmail",
+ scopes: null,
+ username: null,
+ },
+]);
+
+const WORK = "[Work Gmail](credential://google/work-credential-id)";
+const PERSONAL = "[Personal](credential://google/personal-credential-id)";
+
+function Composer({
+ onSend,
+ initialValue = "",
+ rejectEdits = false,
+ onMultilineChange,
+}: {
+ onSend: (message: string) => void;
+ initialValue?: string;
+ rejectEdits?: boolean;
+ onMultilineChange?: (multiline: boolean) => void;
+}) {
+ const [value, setValue] = useState(initialValue);
+ const mentions = useChatMentions({
+ enabled: true,
+ value,
+ setValue,
+ integrations: accounts,
+ includeWorkspaceFiles: false,
+ addWorkspaceFile: vi.fn(),
+ addWorkspaceFolder: vi.fn(),
+ });
+ return (
+ <form
+ onSubmit={(event) => {
+ event.preventDefault();
+ onSend(String(new FormData(event.currentTarget).get("message")));
+ }}
+ >
+ <CredentialMentionEditor
+ value={value}
+ onInputReady={mentions.bindInput}
+ onChange={(next, input) => {
+ if (rejectEdits) return;
+ setValue(next);
+ mentions.detect(input);
+ }}
+ onKeyDown={(event) => {
+ mentions.onKeyDown(event);
+ }}
+ onMultilineChange={onMultilineChange}
+ />
+ <button type="submit">Send</button>
+ </form>
+ );
+}
+
+function placeCaret(node: Node, offset: number) {
+ const range = document.createRange();
+ range.setStart(node, offset);
+ range.collapse(true);
+ const selection = window.getSelection();
+ selection?.removeAllRanges();
+ selection?.addRange(range);
+}
+
+function badgesIn(editor: HTMLElement) {
+ return Array.from(editor.querySelectorAll("[data-credential-mention]"));
+}
+
+describe("account badge composer", () => {
+ it("inserts a visible badge and submits the corresponding credential ID", async () => {
+ const onSend = vi.fn();
+ render(<Composer onSend={onSend} />);
+ fireEvent.change(screen.getByRole("textbox"), {
+ target: { value: "Check @Work", selectionStart: 11 },
+ });
+ fireEvent.keyDown(screen.getByRole("textbox"), { key: "Enter" });
+ const editor = screen.getByRole("textbox");
+ await waitFor(() => expect(editor.textContent).toBe("Check Work Gmail "));
+ expect(editor.textContent).not.toContain("credential-id");
+ fireEvent.click(screen.getByRole("button", { name: "Send" }));
+ expect(onSend).toHaveBeenCalledWith(
+ "Check [Work Gmail](credential://google/work-credential-id) ",
+ );
+ });
+
+ it("removes an account reference when its badge is deleted", async () => {
+ const onSend = vi.fn();
+ render(<Composer onSend={onSend} />);
+ fireEvent.change(screen.getByRole("textbox"), {
+ target: { value: "Check @Work", selectionStart: 11 },
+ });
+ fireEvent.keyDown(screen.getByRole("textbox"), { key: "Enter" });
+ const editor = screen.getByRole("textbox");
+ await waitFor(() => expect(editor.textContent).toContain("Work Gmail"));
+ editor.querySelector("[data-credential-mention]")?.remove();
+ fireEvent.input(editor);
+ fireEvent.click(screen.getByRole("button", { name: "Send" }));
+ expect(onSend).toHaveBeenCalledWith("Check ");
+ });
+
+ it("keeps the caret at the deletion site when the last badge is removed mid-text", async () => {
+ const onSend = vi.fn();
+ const user = userEvent.setup();
+ render(<Composer onSend={onSend} initialValue={`Prefix ${WORK} suffix`} />);
+ const editor = screen.getByRole("textbox");
+ expect(editor.textContent).toBe("Prefix Work Gmail suffix");
+ editor.focus();
+ badgesIn(editor)[0].remove();
+ placeCaret(editor.firstChild!, "Prefix ".length);
+ fireEvent.input(editor);
+
+ const textarea = screen.getByRole<HTMLTextAreaElement>("textbox");
+ expect(textarea.tagName).toBe("TEXTAREA");
+ expect(textarea.value).toBe("Prefix suffix");
+ expect(document.activeElement).toBe(textarea);
+ expect(textarea.selectionStart).toBe("Prefix ".length);
+ expect(textarea.selectionEnd).toBe("Prefix ".length);
+
+ await user.keyboard("X");
+ fireEvent.click(screen.getByRole("button", { name: "Send" }));
+ expect(onSend).toHaveBeenCalledWith("Prefix X suffix");
+ });
+
+ it("keeps the caret after text typed over a selection spanning the last badge", () => {
+ const onSend = vi.fn();
+ render(<Composer onSend={onSend} initialValue={`Prefix ${WORK} suffix`} />);
+ const editor = screen.getByRole("textbox");
+ editor.focus();
+ editor.replaceChildren(
+ document.createTextNode("PreX"),
+ document.createTextNode("fix"),
+ );
+ placeCaret(editor.firstChild!, "PreX".length);
+ fireEvent.input(editor);
+
+ const textarea = screen.getByRole<HTMLTextAreaElement>("textbox");
+ expect(textarea.tagName).toBe("TEXTAREA");
+ expect(textarea.value).toBe("PreXfix");
+ expect(textarea.selectionStart).toBe("PreX".length);
+ expect(textarea.selectionEnd).toBe("PreX".length);
+ });
+
+ it("restores the caret when typing a reference turns the textarea into a rich editor", () => {
+ render(<Composer onSend={vi.fn()} initialValue="Check now" />);
+ const textarea = screen.getByRole<HTMLTextAreaElement>("textbox");
+ textarea.focus();
+ fireEvent.change(textarea, {
+ target: {
+ value: `Check ${WORK} now`,
+ selectionStart: `Check ${WORK}`.length,
+ selectionEnd: `Check ${WORK}`.length,
+ },
+ });
+ const editor = screen.getByRole("textbox");
+ expect(editor.tagName).toBe("DIV");
+ expect(editor.textContent).toBe("Check Work Gmail now");
+ expect(document.activeElement).toBe(editor);
+ const range = window.getSelection()!.getRangeAt(0);
+ expect(range.collapsed).toBe(true);
+ expect(range.comparePoint(badgesIn(editor)[0], 0)).toBe(-1);
+ expect(range.comparePoint(editor.lastChild!, 1)).toBe(1);
+ });
+
+ it("puts back the previous text when the parent rejects an edit", () => {
+ const onSend = vi.fn();
+ render(
+ <Composer onSend={onSend} initialValue={`Check ${WORK} `} rejectEdits />,
+ );
+ const editor = screen.getByRole("textbox");
+ editor.focus();
+ const trailing = editor.lastChild as Text;
+ trailing.textContent += "typed while recording";
+ placeCaret(trailing, trailing.length);
+ fireEvent.input(editor);
+
+ expect(editor.textContent).toBe("Check Work Gmail ");
+ expect(badgesIn(editor)).toHaveLength(1);
+ fireEvent.click(screen.getByRole("button", { name: "Send" }));
+ expect(onSend).toHaveBeenCalledWith(`Check ${WORK} `);
+ });
+
+ it("releases the multiline layout when the last badge goes and the textarea takes over", () => {
+ const onMultilineChange = vi.fn();
+ render(
+ <Composer
+ onSend={vi.fn()}
+ initialValue={`Hi ${WORK}`}
+ onMultilineChange={onMultilineChange}
+ />,
+ );
+ expect(onMultilineChange).toHaveBeenLastCalledWith(true);
+ const editor = screen.getByRole("textbox");
+ badgesIn(editor)[0].remove();
+ fireEvent.input(editor);
+
+ expect(screen.getByRole("textbox").tagName).toBe("TEXTAREA");
+ expect(onMultilineChange).toHaveBeenLastCalledWith(false);
+ });
+
+ it("turns a reference pasted into a rich editor into a badge without exposing its ID", () => {
+ const onSend = vi.fn();
+ render(<Composer onSend={onSend} initialValue={`Check ${WORK} `} />);
+ const editor = screen.getByRole("textbox");
+ editor.focus();
+ const trailing = editor.lastChild as Text;
+ trailing.textContent += PERSONAL;
+ placeCaret(trailing, trailing.length);
+ fireEvent.input(editor);
+
+ const badges = badgesIn(editor);
+ expect(badges).toHaveLength(2);
+ expect(editor.textContent).toBe("Check Work Gmail Personal");
+ expect(editor.textContent).not.toContain("credential");
+ const range = window.getSelection()!.getRangeAt(0);
+ expect(range.collapsed).toBe(true);
+ expect(editor.contains(range.startContainer)).toBe(true);
+ expect(range.comparePoint(badges[1], 0)).toBe(-1);
+ fireEvent.click(screen.getByRole("button", { name: "Send" }));
+ expect(onSend).toHaveBeenCalledWith(`Check ${WORK} ${PERSONAL}`);
+ });
+
+ it("defers turning a reference into a badge until IME composition ends", () => {
+ render(<Composer onSend={vi.fn()} initialValue={`Check ${WORK} `} />);
+ const editor = screen.getByRole("textbox");
+ editor.focus();
+ const trailing = editor.lastChild as Text;
+ fireEvent.compositionStart(editor);
+ trailing.textContent += PERSONAL;
+ placeCaret(trailing, trailing.length);
+ fireEvent.input(editor);
+ expect(badgesIn(editor)).toHaveLength(1);
+ expect(editor.lastChild).toBe(trailing);
+
+ fireEvent.compositionEnd(editor);
+ expect(badgesIn(editor)).toHaveLength(2);
+ expect(editor.textContent).toBe("Check Work Gmail Personal");
+ });
+
+ it("submits one newline for a browser-inserted empty line", () => {
+ const onSend = vi.fn();
+ render(<Composer onSend={onSend} initialValue={`Check ${WORK}`} />);
+ const editor = screen.getByRole("textbox");
+ editor.focus();
+ const line = document.createElement("div");
+ line.appendChild(document.createElement("br"));
+ editor.appendChild(line);
+ placeCaret(line, 0);
+ fireEvent.input(editor);
+ fireEvent.click(screen.getByRole("button", { name: "Send" }));
+ expect(onSend).toHaveBeenCalledWith(`Check ${WORK}\n`);
+ });
+});
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/CredentialMention/__tests__/CredentialMentionMarkdown.test.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/CredentialMention/__tests__/CredentialMentionMarkdown.test.tsx
new file mode 100644
index 0000000000..a9dc749cf7
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/CredentialMention/__tests__/CredentialMentionMarkdown.test.tsx
@@ -0,0 +1,40 @@
+import { render, screen } from "@/tests/integrations/test-utils";
+import { describe, expect, it } from "vitest";
+import { CredentialMentionMarkdown } from "../CredentialMentionMarkdown";
+
+describe("credential badges in messages", () => {
+ it("shows names and logos in place of credential references without losing surrounding markdown", async () => {
+ const { container } = render(
+ <CredentialMentionMarkdown>
+ {
+ "Check **today** in [Work Gmail](credential://google/work-secret-id) and [Personal Gmail](credential://google/personal-secret-id)."
+ }
+ </CredentialMentionMarkdown>,
+ );
+ expect(await screen.findByText("Work Gmail")).toBeTruthy();
+ expect(screen.getByText("Personal Gmail")).toBeTruthy();
+ expect(screen.getByText("today").classList.contains("font-semibold")).toBe(
+ true,
+ );
+ expect(container.textContent).not.toContain("secret-id");
+ expect(container.textContent).not.toContain("credential://");
+ expect(container.querySelectorAll('img[src*="google"]')).toHaveLength(2);
+ expect(container.querySelectorAll("a")).toHaveLength(0);
+ });
+
+ it("keeps the scrollable table wrapper from the message renderer", async () => {
+ const { container } = render(
+ <CredentialMentionMarkdown>
+ {
+ "| Account | Status |\n| --- | --- |\n| [Work Gmail](credential://google/work-secret-id) | ok |"
+ }
+ </CredentialMentionMarkdown>,
+ );
+ expect(await screen.findByText("Work Gmail")).toBeTruthy();
+ const table = container.querySelector("table");
+ expect(table).not.toBeNull();
+ expect(table?.parentElement?.className).toBe(
+ "my-4 max-w-full overflow-x-auto",
+ );
+ });
+});
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/CredentialMention/__tests__/editorHelpers.test.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/components/CredentialMention/__tests__/editorHelpers.test.ts
new file mode 100644
index 0000000000..ff1801313a
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/CredentialMention/__tests__/editorHelpers.test.ts
@@ -0,0 +1,82 @@
+import { describe, expect, it } from "vitest";
+import { parseCredentialMentions } from "../helpers";
+import {
+ editorSelectionOffset,
+ readMentionEditor,
+ setEditorCaret,
+} from "../editorHelpers";
+
+const BADGE = `<span contenteditable="false" data-credential-mention="[Work](credential://google/work-id)"></span>`;
+
+function editorWith(html: string) {
+ const root = document.createElement("div");
+ root.contentEditable = "true";
+ root.innerHTML = html;
+ document.body.appendChild(root);
+ return root;
+}
+
+describe("editor newline serialization", () => {
+ it.each([
+ ["Hello<div><br></div>", "Hello\n"],
+ ["First<br>Second", "First\nSecond"],
+ ["<div>First</div><div>Second</div>", "First\nSecond"],
+ ["<div>A</div><div><br></div><div>B</div>", "A\n\nB"],
+ ["<p>A</p><p>B</p>", "A\nB"],
+ ["A<br><br>", "A\n"],
+ ["<br>", ""],
+ [
+ `Check ${BADGE}<div>next</div>`,
+ "Check [Work](credential://google/work-id)\nnext",
+ ],
+ ])("serializes %s as %j", (html, expected) => {
+ expect(readMentionEditor(editorWith(html))).toBe(expected);
+ });
+});
+
+function snapPastBadge(text: string, offset: number) {
+ let position = 0;
+ for (const part of parseCredentialMentions(text)) {
+ const length = typeof part === "string" ? part.length : part.token.length;
+ if (
+ typeof part !== "string" &&
+ offset > position &&
+ offset < position + length
+ )
+ return position + length;
+ position += length;
+ }
+ return offset;
+}
+
+describe("editor caret round trip", () => {
+ it.each([
+ "First<br>Second",
+ "<div>First</div><div>Second</div>",
+ "<div>A</div><div><br></div><div>B</div>",
+ "<p>A</p><p>B</p>",
+ "Hello<div><br></div>",
+ `Check ${BADGE} then<div>next ${BADGE}</div>`,
+ ])("reads back every caret offset in %s", (html) => {
+ const root = editorWith(html);
+ const text = readMentionEditor(root);
+ for (let offset = 0; offset <= text.length; offset++) {
+ setEditorCaret(root, offset);
+ expect(editorSelectionOffset(root), `offset ${offset}`).toBe(
+ snapPastBadge(text, offset),
+ );
+ }
+ });
+
+ it("selects a serialized range across a badge", () => {
+ const root = editorWith(`Check ${BADGE} then`);
+ setEditorCaret(
+ root,
+ 2,
+ "Check [Work](credential://google/work-id) th".length,
+ );
+ const range = window.getSelection()!.getRangeAt(0);
+ expect(range.collapsed).toBe(false);
+ expect(range.toString()).toBe("eck th");
+ });
+});
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/CredentialMention/__tests__/helpers.test.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/components/CredentialMention/__tests__/helpers.test.ts
new file mode 100644
index 0000000000..a016cfa048
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/CredentialMention/__tests__/helpers.test.ts
@@ -0,0 +1,47 @@
+import { describe, expect, it } from "vitest";
+import {
+ parseCredentialMentions,
+ serializeCredentialMention,
+ credentialMentionDisplayText,
+} from "../helpers";
+
+const account = {
+ credentialId: "account-work",
+ provider: "google",
+ name: "Work [Gmail]",
+};
+
+describe("credential references", () => {
+ it("round trips the exact account ID and escaped label", () => {
+ const token = serializeCredentialMention(account);
+ expect(token).toContain("credential://google/account-work");
+ expect(parseCredentialMentions(`Check ${token} now`)).toEqual([
+ "Check ",
+ { ...account, token },
+ " now",
+ ]);
+ });
+
+ it("keeps identical names associated with different accounts", () => {
+ expect(serializeCredentialMention(account)).not.toBe(
+ serializeCredentialMention({
+ ...account,
+ credentialId: "account-personal",
+ }),
+ );
+ });
+
+ it("copies visible labels without exposing IDs", () => {
+ expect(
+ credentialMentionDisplayText(
+ `Check ${serializeCredentialMention(account)}`,
+ ),
+ ).toBe("Check Work [Gmail]");
+ });
+
+ it("treats malformed encoding as text", () => {
+ expect(parseCredentialMentions("[Work](credential://google/%ZZ)")).toEqual([
+ "[Work](credential://google/%ZZ)",
+ ]);
+ });
+});
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/CredentialMention/editorHelpers.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/components/CredentialMention/editorHelpers.ts
new file mode 100644
index 0000000000..a0b9d839d0
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/CredentialMention/editorHelpers.ts
@@ -0,0 +1,152 @@
+import type { MentionInput } from "../ChatInput/useChatMentions";
+
+/** One serialized run of the editor DOM: a text node, a badge (which
+ * serializes as its credential token), or a line break. A break is either a
+ * BR or a block element whose start implies a newline after its predecessor.
+ * A BR that is the last child of its block is the placeholder browsers keep
+ * so an empty line has height; it is not a newline of its own. */
+interface Piece {
+ node: Node;
+ text: string;
+ kind: "text" | "token" | "break";
+}
+
+function isBlock(node: Node) {
+ return node.nodeName === "DIV" || node.nodeName === "P";
+}
+
+function editorPieces(node: Node, pieces: Piece[] = []): Piece[] {
+ if (node instanceof HTMLElement && node.dataset.credentialMention) {
+ pieces.push({ node, text: node.dataset.credentialMention, kind: "token" });
+ return pieces;
+ }
+ if (node.nodeType === Node.TEXT_NODE) {
+ pieces.push({ node, text: node.textContent ?? "", kind: "text" });
+ return pieces;
+ }
+ if (node.nodeName === "BR") {
+ if (node.nextSibling) pieces.push({ node, text: "\n", kind: "break" });
+ return pieces;
+ }
+ let previous: Node | null = null;
+ for (const child of Array.from(node.childNodes)) {
+ if (previous && (isBlock(child) || isBlock(previous)))
+ pieces.push({ node: child, text: "\n", kind: "break" });
+ editorPieces(child, pieces);
+ previous = child;
+ }
+ return pieces;
+}
+
+export function readMentionEditor(node: Node): string {
+ return editorPieces(node)
+ .map((piece) => piece.text)
+ .join("");
+}
+
+/** Serialized text runs between badges, so a credential reference typed or
+ * pasted as plain text can be found even when the browser split it across
+ * text nodes or lines. */
+export function editorTextRuns(root: HTMLElement): string[] {
+ const runs: string[] = [];
+ let current = "";
+ for (const piece of editorPieces(root)) {
+ if (piece.kind === "token") {
+ runs.push(current);
+ current = "";
+ } else current += piece.text;
+ }
+ runs.push(current);
+ return runs;
+}
+
+function pointOffset(root: HTMLElement, container: Node, offset: number) {
+ const point = document.createRange();
+ point.setStart(container, offset);
+ point.collapse(true);
+ let position = 0;
+ for (const piece of editorPieces(root)) {
+ if (piece.kind === "text" && piece.node === container)
+ return position + Math.min(offset, piece.text.length);
+ if (point.comparePoint(piece.node, 0) === 1) return position;
+ position += piece.text.length;
+ }
+ return position;
+}
+
+export function editorSelectionRange(root: HTMLElement) {
+ const selection = window.getSelection();
+ if (!selection?.rangeCount) return null;
+ const range = selection.getRangeAt(0);
+ if (
+ !root.contains(range.startContainer) ||
+ !root.contains(range.endContainer)
+ )
+ return null;
+ return {
+ start: pointOffset(root, range.startContainer, range.startOffset),
+ end: pointOffset(root, range.endContainer, range.endOffset),
+ };
+}
+
+export function editorSelectionOffset(root: HTMLElement) {
+ return editorSelectionRange(root)?.start ?? readMentionEditor(root).length;
+}
+
+function pointAt(root: HTMLElement, offset: number): Range {
+ const range = document.createRange();
+ let position = 0;
+ for (const piece of editorPieces(root)) {
+ const end = position + piece.text.length;
+ if (piece.kind === "text" && offset <= end) {
+ range.setStart(piece.node, Math.max(0, offset - position));
+ return range;
+ }
+ if (piece.kind === "token" && offset === position) {
+ range.setStartBefore(piece.node);
+ return range;
+ }
+ if (piece.kind === "token" && offset < end) {
+ range.setStartAfter(piece.node);
+ return range;
+ }
+ if (piece.kind === "break" && offset === position) {
+ range.setStartBefore(piece.node);
+ return range;
+ }
+ if (piece.kind === "break" && offset === end) {
+ if (piece.node.nodeName === "BR") range.setStartAfter(piece.node);
+ else range.setStart(piece.node, 0);
+ return range;
+ }
+ position = end;
+ }
+ range.selectNodeContents(root);
+ range.collapse(false);
+ return range;
+}
+
+export function setEditorCaret(root: HTMLElement, start: number, end = start) {
+ const from = pointAt(root, Math.max(0, start));
+ const to = pointAt(root, Math.max(start, end));
+ const range = document.createRange();
+ range.setStart(from.startContainer, from.startOffset);
+ range.setEnd(to.startContainer, to.startOffset);
+ const selection = window.getSelection();
+ selection?.removeAllRanges();
+ selection?.addRange(range);
+}
+
+export function mentionInputFor(root: HTMLElement): MentionInput {
+ return {
+ get value() {
+ return readMentionEditor(root);
+ },
+ get selectionStart() {
+ return editorSelectionOffset(root);
+ },
+ setSelectionRange(start: number, end: number) {
+ setEditorCaret(root, start, end);
+ },
+ };
+}
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/CredentialMention/helpers.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/components/CredentialMention/helpers.ts
new file mode 100644
index 0000000000..207cb0cefc
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/CredentialMention/helpers.ts
@@ -0,0 +1,70 @@
+export interface CredentialMention {
+ credentialId: string;
+ provider: string;
+ name: string;
+}
+
+export interface CredentialMentionPart extends CredentialMention {
+ token: string;
+}
+
+export const CREDENTIAL_IMAGE_PREFIX = "/__credential-mention/";
+const MENTION_PATTERN =
+ /\[((?:\\.|[^\\\]])*)\]\(credential:\/\/([^/\s]+)\/([^\s)]+)\)/g;
+
+export function serializeCredentialMention(mention: CredentialMention) {
+ const name = mention.name.replace(/[\\[\]]/g, "\\$&").replace(/\r?\n/g, " ");
+ return `[${name}](credential://${encodeURIComponent(mention.provider)}/${encodeURIComponent(mention.credentialId)})`;
+}
+
+export function parseCredentialMentions(
+ text: string,
+): (string | CredentialMentionPart)[] {
+ const parts: (string | CredentialMentionPart)[] = [];
+ let end = 0;
+ for (const match of text.matchAll(MENTION_PATTERN)) {
+ let mention: CredentialMentionPart;
+ try {
+ mention = {
+ name: match[1].replace(/\\(.)/g, "$1"),
+ provider: decodeURIComponent(match[2]),
+ credentialId: decodeURIComponent(match[3]),
+ token: match[0],
+ };
+ } catch {
+ continue;
+ }
+ if (match.index > end) parts.push(text.slice(end, match.index));
+ parts.push(mention);
+ end = match.index + match[0].length;
+ }
+ if (end < text.length) parts.push(text.slice(end));
+ return parts;
+}
+
+export function credentialMentionDisplayText(text: string) {
+ return parseCredentialMentions(text)
+ .map((part) => (typeof part === "string" ? part : part.name))
+ .join("");
+}
+
+export function credentialMentionsToMarkdown(text: string) {
+ return parseCredentialMentions(text)
+ .map((part) =>
+ typeof part === "string"
+ ? part
+ : `![${part.name.replace(/[\\[\]]/g, "\\$&")}](${CREDENTIAL_IMAGE_PREFIX}${encodeURIComponent(part.provider)}/${encodeURIComponent(part.credentialId)})`,
+ )
+ .join("");
+}
+
+export function credentialMentionImageProvider(src?: string) {
+ if (!src?.startsWith(CREDENTIAL_IMAGE_PREFIX)) return null;
+ try {
+ return decodeURIComponent(
+ src.slice(CREDENTIAL_IMAGE_PREFIX.length).split("/")[0],
+ );
+ } catch {
+ return null;
+ }
+}
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/CredentialMention/useCredentialMentionEditor.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/components/CredentialMention/useCredentialMentionEditor.ts
new file mode 100644
index 0000000000..af53583d02
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/CredentialMention/useCredentialMentionEditor.ts
@@ -0,0 +1,96 @@
+import {
+ useCallback,
+ useLayoutEffect,
+ useReducer,
+ useRef,
+ useState,
+} from "react";
+import { parseCredentialMentions, type CredentialMentionPart } from "./helpers";
+import {
+ editorSelectionOffset,
+ editorTextRuns,
+ readMentionEditor,
+ setEditorCaret,
+} from "./editorHelpers";
+
+interface BadgeMount {
+ element: HTMLSpanElement;
+ mention: CredentialMentionPart;
+}
+
+/** True while the DOM already shows `value` with every credential reference
+ * as a badge. A reference typed or pasted as plain text serializes to the
+ * same value, so the text runs between badges are checked as well. */
+function editorShows(editor: HTMLElement, value: string) {
+ return (
+ readMentionEditor(editor) === value &&
+ !editorTextRuns(editor).some((run) =>
+ parseCredentialMentions(run).some((part) => typeof part !== "string"),
+ )
+ );
+}
+
+export function useCredentialMentionEditor(
+ value: string,
+ onMultilineChange?: (multiline: boolean) => void,
+) {
+ const editorRef = useRef<HTMLDivElement | null>(null);
+ const [badges, setBadges] = useState<BadgeMount[]>([]);
+ const composingRef = useRef(false);
+ const [compositionCount, setCompositionCount] = useState(0);
+ // The div is uncontrolled, so an edit the parent rejects (it kept `value`)
+ // would otherwise stay in the DOM; bumping this re-runs the comparison.
+ const [resyncCount, resync] = useReducer((count: number) => count + 1, 0);
+ const onMultilineRef = useRef(onMultilineChange);
+ onMultilineRef.current = onMultilineChange;
+
+ // Stable so React does not detach and re-attach it on every render. The
+ // textarea that replaces the editor only reports wrapping it measures, so
+ // the stacked layout is released here, in the mutation phase before the
+ // textarea's layout effect can claim it again.
+ const attachEditor = useCallback((node: HTMLDivElement | null) => {
+ if (!node && editorRef.current) onMultilineRef.current?.(false);
+ editorRef.current = node;
+ }, []);
+
+ useLayoutEffect(() => {
+ const editor = editorRef.current;
+ if (!editor || composingRef.current || editorShows(editor, value)) return;
+ const focused = document.activeElement === editor;
+ const caret = editorSelectionOffset(editor);
+ const mounts: BadgeMount[] = [];
+ const nodes = parseCredentialMentions(value).map((part) => {
+ if (typeof part === "string") return document.createTextNode(part);
+ const element = document.createElement("span");
+ element.contentEditable = "false";
+ element.dataset.credentialMention = part.token;
+ mounts.push({ element, mention: part });
+ return element;
+ });
+ editor.replaceChildren(...nodes);
+ setBadges(mounts);
+ if (focused) setEditorCaret(editor, Math.min(caret, value.length));
+ }, [value, compositionCount, resyncCount]);
+
+ useLayoutEffect(() => {
+ if (editorRef.current) onMultilineRef.current?.(true);
+ }, [value]);
+
+ function onCompositionStart() {
+ composingRef.current = true;
+ }
+
+ function onCompositionEnd() {
+ composingRef.current = false;
+ setCompositionCount((count) => count + 1);
+ }
+
+ return {
+ editorRef,
+ attachEditor,
+ badges,
+ resync,
+ onCompositionStart,
+ onCompositionEnd,
+ };
+}
diff --git a/autogpt_platform/frontend/src/components/ai-elements/message.tsx b/autogpt_platform/frontend/src/components/ai-elements/message.tsx
index d63a458640..e6ddb6de89 100644
--- a/autogpt_platform/frontend/src/components/ai-elements/message.tsx
+++ b/autogpt_platform/frontend/src/components/ai-elements/message.tsx
@@ -357,7 +357,7 @@ function ExternalLinkModal({
const math = createMathPlugin({ singleDollarTextMath: true });
export const MessageResponse = memo(
- ({ className, children, ...props }: MessageResponseProps) => (
+ ({ className, children, components, ...props }: MessageResponseProps) => (
<Streamdown
className={cn(
"size-full [&>*:first-child]:mt-0 [&>*:last-child]:mb-0 [&_pre]:!bg-white",
@@ -381,6 +381,7 @@ export const MessageResponse = memo(
<table {...tableProps}>{children}</table>
</div>
),
+ ...components,
}}
plugins={{ code, mermaid, math, cjk }}
linkSafety={{