Skip to content

A personal agent with a browser, terminal, files, and work that keeps going built with CopilotKit and AG-UI.

TypeScript
23.9 KB
Raw Blame
import { randomUUID } from "node:crypto";
import {
  type ChatCompletionRequest,
  type ChatMessage,
  type FixtureResponse,
  getTextContent,
  LLMock,
} from "@copilotkit/aimock";
import { z } from "zod";
import { aquariumFixture, schoolTripFixture } from "./jev-fixture.ts";

export const demoModel = "openai/openmuse-browser-demo";

const pageSchema = z.object({
  sessionId: z.string().min(1),
  url: z.url(),
  title: z.string(),
  text: z.string(),
  truncated: z.boolean(),
});

function targetUrl(prompt: string): string | undefined {
  if (/monterey|aquarium/i.test(prompt))
    return "https://www.montereybayaquarium.org/visit/exhibits";
  if (/copilotkit\.ai/i.test(prompt)) return "https://copilotkit.ai";
  if (/hacker\s*news|news\.ycombinator\.com|cool stuff/i.test(prompt))
    return "https://news.ycombinator.com";
  return undefined;
}

function summarizePage(message: ChatMessage): FixtureResponse {
  let value: unknown;
  try {
    value = JSON.parse(getTextContent(message.content) ?? "");
  } catch {
    return { content: "The browser did not return readable page data. Check the browser result." };
  }
  const parsed = pageSchema.safeParse(value);
  if (!parsed.success)
    return { content: "The browser could not read that page. Check the browser result and retry." };
  const page = parsed.data;
  const lines = page.text
    .split(/\n+/)
    .map((line) => line.trim())
    .filter(Boolean);
  let excerpts: string[];
  let introduction: string;
  if (new URL(page.url).hostname === "news.ycombinator.com") {
    excerpts = lines
      .map(
        (line, index) =>
          /^\d+\.\s+(.+)$/.exec(line)?.[1] ?? (/^\d+\.$/.test(line) ? lines[index + 1] : undefined),
      )
      .filter((line): line is string => Boolean(line))
      .slice(0, 3);
    introduction = "From the current Hacker News front page:";
  } else if (new URL(page.url).hostname.endsWith("montereybayaquarium.org")) {
    excerpts = lines
      .flatMap((line, index) => {
        if (lines[index - 1] !== "EXHIBIT" || !/^(Kelp Forest|Open Sea|Sea Otters)$/i.test(line))
          return [];
        const description = lines[index + 1];
        return [
          description && description !== "Explore exhibit" ? `${line}: ${description}` : line,
        ];
      })
      .slice(0, 3);
    introduction = "Exhibits from the aquarium’s own guide:";
  } else {
    excerpts = lines
      .filter((line) => line.length >= 45 && /agent|copilotkit|ag.ui|framework/i.test(line))
      .slice(0, 3);
    introduction = "From CopilotKit’s current page:";
  }
  if (!excerpts.length) {
    excerpts = lines.filter((line) => line.length >= 30).slice(0, 3);
    introduction = "Here are excerpts from the page I just opened:";
  }
  if (!excerpts.length)
    return { content: "The page opened, but it did not expose enough readable text to summarize." };
  const bullets = excerpts.map(
    (line) => `• ${line.length > 110 ? `${line.slice(0, 110).replace(/\s+\S*$/, "")}…` : line}`,
  );
  return {
    content: `${introduction}\n\n${bullets.join("\n")}\n\nSource: ${page.url}${page.truncated ? "\nThe browser returned a shortened page extract." : ""}`,
  };
}

function turnResult(turn: ChatMessage[], name: string, prefix: string) {
  const ids = new Set(
    turn.flatMap((message) =>
      (message.tool_calls ?? [])
        .filter((call) => call.function.name === name)
        .map((call) => call.id),
    ),
  );
  return turn.findLast(
    (message) =>
      message.role === "tool" &&
      message.tool_call_id &&
      (ids.has(message.tool_call_id) || message.tool_call_id.startsWith(prefix)),
  );
}

function parseResult(message: ChatMessage): unknown {
  try {
    return JSON.parse(getTextContent(message.content) ?? "");
  } catch {
    return undefined;
  }
}

const schoolMessageSchema = z.object({
  id: z.string().optional(),
  sender: z.string(),
  subject: z.string(),
  body: z.string(),
});

function isSchoolTripMessage(message: z.infer<typeof schoolMessageSchema>): boolean {
  return (
    /lincoln middle school/i.test(message.sender) &&
    /permission|trip|aquarium/i.test(message.subject) &&
    /aquarium/i.test(message.body) &&
    /permission slip/i.test(message.body)
  );
}

function schoolMessageFromHistory(messages: ChatMessage[]) {
  for (const result of [...messages].reverse()) {
    if (result.role !== "tool") continue;
    const parsed = z
      .object({ messages: z.array(schoolMessageSchema) })
      .safeParse(parseResult(result));
    if (!parsed.success) continue;
    const match = parsed.data.messages.findLast(isSchoolTripMessage);
    if (match) return match;
  }
  return undefined;
}

function demoMailResponse(request: ChatCompletionRequest, turn: ChatMessage[]): FixtureResponse {
  const read = turnResult(turn, "read_mail_thread", "call_openmuse_demo_mail_read_");
  if (read) {
    const parsed = z
      .object({
        messages: z.array(z.object({ sender: z.string(), subject: z.string(), body: z.string() })),
      })
      .safeParse(parseResult(read));
    const message = parsed.success ? parsed.data.messages.at(-1) : undefined;
    if (!message)
      return {
        content: "I couldn’t read the school-trip email. Check the mail result and try again.",
      };
    const paragraphs = message.body
      .split(/\n\s*\n/)
      .map((text) => text.trim())
      .filter((text) => text.length > 40 && !/local workspace/i.test(text))
      .slice(0, 2);
    if (!paragraphs.length)
      return { content: "The email was found, but it did not include readable trip details." };
    return {
      content: `From ${message.sender}:\n“${message.subject}”\n\n${paragraphs.join("\n\n")}\n\nI can look up the aquarium next.`,
    };
  }
  const search = turnResult(turn, "search_mail", "call_openmuse_demo_mail_search_");
  if (search) {
    const parsed = z
      .object({ matches: z.array(z.object({ threadId: z.string(), subject: z.string() })) })
      .safeParse(parseResult(search));
    if (!parsed.success)
      return { content: "I couldn’t check your inbox. Check the mail connection and try again." };
    const match = parsed.data.matches[0];
    if (!match) return { content: "I didn’t find a school-trip email in the connected mailbox." };
    if (!request.tools?.some((tool) => tool.function.name === "read_mail_thread"))
      return {
        content: "The email reader is unavailable. Open Mail to read the matching message.",
      };
    return {
      content: "I found the school’s reminder. I’ll read the details.",
      toolCalls: [
        {
          id: `call_openmuse_demo_mail_read_${randomUUID()}`,
          name: "read_mail_thread",
          arguments: JSON.stringify({ threadId: match.threadId }),
        },
      ],
    };
  }
  if (!request.tools?.some((tool) => tool.function.name === "search_mail"))
    return { content: "Mail search is unavailable. Connect the mailbox before checking email." };
  return {
    content: "I’ll check your inbox for the school trip.",
    toolCalls: [
      {
        id: `call_openmuse_demo_mail_search_${randomUUID()}`,
        name: "search_mail",
        arguments: JSON.stringify({ query: "aquarium" }),
      },
    ],
  };
}

function demoTripResponse(request: ChatCompletionRequest, turn: ChatMessage[]): FixtureResponse {
  const choice = turnResult(turn, "present_choices", "call_openmuse_demo_jev_");
  if (choice) {
    const parsed = z
      .object({ panel: z.object({ id: z.string() }).nullable(), error: z.string().optional() })
      .safeParse(parseResult(choice));
    return parsed.success && parsed.data.panel
      ? { content: "I found the school reminder. Choose what you’d like to work on first." }
      : {
          content:
            "I couldn’t prepare the trip choices. Please try again when choices are available.",
        };
  }
  const read = turnResult(turn, "read_mail_thread", "call_openmuse_demo_mail_read_");
  if (read) {
    const parsed = z
      .object({
        messages: z.array(schoolMessageSchema),
      })
      .safeParse(parseResult(read));
    const message = parsed.success ? parsed.data.messages.findLast(isSchoolTripMessage) : undefined;
    if (!message)
      return { content: "I couldn’t verify the school-trip email. Check Mail and try again." };
    const search = turnResult(turn, "search_mail", "call_openmuse_demo_mail_search_");
    const searched = z
      .object({
        matches: z.array(
          z.object({ threadId: z.string(), sender: z.string(), subject: z.string() }),
        ),
      })
      .safeParse(search ? parseResult(search) : undefined);
    const mailThreadId = searched.success
      ? searched.data.matches.find(
          (candidate) =>
            /lincoln middle school/i.test(candidate.sender) &&
            /permission|trip|aquarium/i.test(candidate.subject),
        )?.threadId
      : undefined;
    if (!mailThreadId)
      return { content: "I couldn’t verify which school-trip thread was read. Please retry." };
    if (!request.tools?.some((tool) => tool.function.name === "present_choices"))
      return {
        content:
          "The choices tool is unavailable. Please try again after enabling Jev sample mode.",
      };
    return {
      content:
        "The school reminder mentions the permission slip and trip details. I’ll lay out the next steps.",
      toolCalls: [
        {
          id: `call_openmuse_demo_jev_${randomUUID()}`,
          name: "present_choices",
          arguments: JSON.stringify({
            message: "Help me get ready for the aquarium trip",
            context: `${schoolTripFixture.school}: ${message.subject}. ${message.body.slice(0, 1200)}`,
            title: "How should we get ready?",
            control: "clarification",
            options: schoolTripFixture.choices,
            mailThreadId,
          }),
        },
      ],
    };
  }
  const search = turnResult(turn, "search_mail", "call_openmuse_demo_mail_search_");
  if (search) {
    const parsed = z
      .object({
        matches: z.array(
          z.object({ threadId: z.string(), sender: z.string(), subject: z.string() }),
        ),
      })
      .safeParse(parseResult(search));
    if (!parsed.success) return { content: "I couldn’t check the mailbox for the school trip." };
    const match = parsed.data.matches.find(
      (candidate) =>
        /lincoln middle school/i.test(candidate.sender) &&
        /permission|trip|aquarium/i.test(candidate.subject),
    );
    if (!match) return { content: "I didn’t find a school-trip email in the sample mailbox." };
    if (!request.tools?.some((tool) => tool.function.name === "read_mail_thread"))
      return { content: "The email reader is unavailable." };
    return {
      content: "I found the school reminder. I’ll read it before suggesting next steps.",
      toolCalls: [
        {
          id: `call_openmuse_demo_mail_read_${randomUUID()}`,
          name: "read_mail_thread",
          arguments: JSON.stringify({ threadId: match.threadId }),
        },
      ],
    };
  }
  if (!request.tools?.some((tool) => tool.function.name === "search_mail"))
    return { content: "Mail search is unavailable. Connect the sample mailbox and try again." };
  return {
    content: "I’ll check the sample school email first.",
    toolCalls: [
      {
        id: `call_openmuse_demo_mail_search_${randomUUID()}`,
        name: "search_mail",
        arguments: JSON.stringify({ query: schoolTripFixture.searchQuery }),
      },
    ],
  };
}

function demoSchoolChoiceResponse(
  request: ChatCompletionRequest,
  turn: ChatMessage[],
  choice: "review" | "complete",
): FixtureResponse {
  const mail = schoolMessageFromHistory(request.messages);
  if (!mail)
    return {
      content:
        "I can’t find the verified school-trip email in this conversation. Please start with the trip request again.",
    };
  if (choice === "review")
    return {
      content: `From ${mail.sender}, “${mail.subject}”:\n\n${mail.body.replace(/\n\nThis message is included with your local workspace\.?/i, "").slice(0, 1200)}`,
    };
  const task = turnResult(turn, "delegate_task", "call_openmuse_demo_document_");
  if (task) {
    const parsed = z.object({ id: z.string().min(1) }).safeParse(parseResult(task));
    return parsed.success
      ? {
          content:
            "I started preparing the permission slip. Follow the document task in Activity; it will ask for form details and review before sending.",
        }
      : { content: "I couldn’t start the permission-slip task. Please try again from the email." };
  }
  if (!mail.id)
    return {
      content:
        "The school-trip email has no message ID, so I can’t start the permission-slip task. Open it in Mail first.",
    };
  if (!request.tools?.some((tool) => tool.function.name === "delegate_task"))
    return {
      content: "The document task tool is unavailable. Open the permission slip from Mail.",
    };
  return {
    content: "I’ll prepare the school’s permission slip for your review.",
    toolCalls: [
      {
        id: `call_openmuse_demo_document_${randomUUID()}`,
        name: "delegate_task",
        arguments: JSON.stringify({
          kind: "document",
          title: "Complete the permission slip",
          prompt: "Complete the permission slip attached to the verified school-trip email",
          input: { messageId: mail.id },
        }),
      },
    ],
  };
}

function previousComparisonId(messages: ChatMessage[]): string | undefined {
  for (const message of [...messages].reverse()) {
    if (message.role !== "tool") continue;
    const parsed = z
      .object({ panel: z.object({ id: z.string(), type: z.literal("comparison") }) })
      .safeParse(parseResult(message));
    if (parsed.success) return parsed.data.panel.id;
  }
  return undefined;
}

function pageExcerpt(text: string, anchor: RegExp): string {
  const line = text.split(/\r?\n/).find((candidate) => anchor.test(candidate));
  const normalized = (line ?? text).replace(/\s+/g, " ").trim();
  const match = anchor.exec(normalized);
  if (!match) throw new Error("The exhibit page is missing its verified detail");
  const previousSentence = normalized.lastIndexOf(". ", match.index);
  const start = previousSentence >= 0 ? previousSentence + 2 : 0;
  const nextStop = normalized.slice(match.index + match[0].length).search(/[.!?](?:\s|$)/);
  const sentenceEnd =
    nextStop >= 0 ? match.index + match[0].length + nextStop + 1 : normalized.length;
  if (sentenceEnd - start <= 260) return normalized.slice(start, sentenceEnd).trim();
  const end = Math.min(normalized.length, match.index + match[0].length + 120);
  const wordEnd = end < normalized.length ? normalized.lastIndexOf(" ", end) : end;
  return normalized.slice(start, wordEnd > start ? wordEnd : end).trim();
}

function demoExhibitResponse(request: ChatCompletionRequest, turn: ChatMessage[]): FixtureResponse {
  const choice = turnResult(turn, "present_choices", "call_openmuse_demo_jev_");
  if (choice) {
    const parsed = z
      .object({ panel: z.object({ id: z.string() }).nullable(), error: z.string().optional() })
      .safeParse(parseResult(choice));
    return parsed.success && parsed.data.panel
      ? {
          content: "Here are three researched exhibits. Tell me what matters most, or choose one.",
        }
      : { content: "I couldn’t prepare the exhibit choices. Please retry." };
  }
  const browseResults = turn.filter(
    (message) =>
      message.role === "tool" &&
      message.tool_call_id?.startsWith("call_openmuse_demo_exhibit_browse_"),
  );
  const observed = new Set<string>();
  const observedPages = new Map<string, string>();
  const requiredEvidence: Record<string, RegExp[]> = {
    "kelp-forest": [
      /kelp forest/i,
      /28\s*(?:feet|foot)|28-foot/i,
      /sardines?/i,
      /leopard sharks?/i,
    ],
    "open-sea": [/open sea/i, /90-foot|90\s*foot/i, /turtles?/i, /sardines?/i, /tuna/i],
    "rocky-shore": [/rocky shore/i, /bat rays?/i, /touch pool/i],
  };
  for (const result of browseResults) {
    const parsed = pageSchema.safeParse(parseResult(result));
    if (!parsed.success || !parsed.data.text.trim())
      return {
        content:
          "I couldn’t read an aquarium exhibit page, so I can’t prepare sourced choices yet.",
      };
    const source = aquariumFixture.options.find(
      (option) => option.sources[0].url.replace(/\/$/, "") === parsed.data.url.replace(/\/$/, ""),
    );
    if (!source || !requiredEvidence[source.id].every((pattern) => pattern.test(parsed.data.text)))
      return {
        content:
          "I couldn’t verify the exhibit details in the aquarium page, so I can’t prepare sourced choices yet.",
      };
    observed.add(parsed.data.url.replace(/\/$/, ""));
    observedPages.set(source.id, parsed.data.text);
  }
  const next = aquariumFixture.options.find(
    (option) => !observed.has(option.sources[0].url.replace(/\/$/, "")),
  );
  if (next) {
    if (!request.tools?.some((tool) => tool.function.name === "browse_web"))
      return { content: "The browser is unavailable, so I can’t research the exhibits." };
    return {
      content: `I’ll read the aquarium’s ${next.label} page.`,
      toolCalls: [
        {
          id: `call_openmuse_demo_exhibit_browse_${randomUUID()}`,
          name: "browse_web",
          arguments: JSON.stringify({ url: next.sources[0].url }),
        },
      ],
    };
  }
  if (!request.tools?.some((tool) => tool.function.name === "present_choices"))
    return {
      content: "The choices tool is unavailable. Please try again after enabling Jev sample mode.",
    };
  const options =
    process.env.DEMO_JEV_MODE === "live"
      ? aquariumFixture.options.map((option) => {
          const text = observedPages.get(option.id);
          if (!text) throw new Error("A verified aquarium page is missing");
          const anchor =
            option.id === "kelp-forest"
              ? /28\s*(?:feet|foot)|28-foot/i
              : option.id === "open-sea"
                ? /90-foot|90\s*foot/i
                : /touch pool/i;
          return {
            ...option,
            details: [pageExcerpt(text, anchor)],
            sources: [{ ...option.sources[0], title: option.label }],
          };
        })
      : aquariumFixture.options;
  return {
    content: "The three aquarium pages are open. I’ll compare their verified exhibit details.",
    toolCalls: [
      {
        id: `call_openmuse_demo_jev_${randomUUID()}`,
        name: "present_choices",
        arguments: JSON.stringify({
          message: "Explore exhibits for the aquarium school trip",
          context:
            "Candidate excerpts from the three official Monterey Bay Aquarium pages read in this turn.",
          title: "Three exhibits to explore",
          control: "comparison",
          options,
        }),
      },
    ],
  };
}

function demoRefinementResponse(
  request: ChatCompletionRequest,
  turn: ChatMessage[],
): FixtureResponse {
  const choice = turnResult(turn, "present_choices", "call_openmuse_demo_jev_");
  if (choice) {
    const parsed = z
      .object({
        panel: z
          .object({ id: z.string(), options: z.array(z.object({ label: z.string() })) })
          .nullable(),
        error: z.string().optional(),
      })
      .safeParse(parseResult(choice));
    return parsed.success && parsed.data.panel
      ? {
          content: `Rocky Shore has the aquarium’s bat-ray touch pool. The updated cards put ${parsed.data.panel.options[0]?.label ?? "an exhibit"} first.`,
        }
      : { content: "I couldn’t refine the exhibit choices. Please retry." };
  }
  const currentUserIndex = request.messages.findLastIndex((message) => message.role === "user");
  const panelId = previousComparisonId(request.messages.slice(0, currentUserIndex));
  if (!panelId)
    return {
      content: "I don’t have an exhibit comparison to refine yet. Choose Explore exhibits first.",
    };
  if (!request.tools?.some((tool) => tool.function.name === "present_choices"))
    return {
      content: "The choices tool is unavailable. Please try again after enabling Jev sample mode.",
    };
  return {
    content: "I’ll rerank the researched exhibits for a hands-on visit.",
    toolCalls: [
      {
        id: `call_openmuse_demo_jev_${randomUUID()}`,
        name: "present_choices",
        arguments: JSON.stringify({
          message: "Something hands-on",
          context:
            "Refine the researched comparison: Rocky Shore offers a bat-ray touch pool; the other two are viewing exhibits.",
          title: "Hands-on exhibit first",
          control: "comparison",
          refinementPanelId: panelId,
          options: aquariumFixture.options,
        }),
      },
    ],
  };
}

/** Script only the model: the app executes real mailbox reads and browser tools. */
export function demoResponse(request: ChatCompletionRequest): FixtureResponse {
  const userIndex = request.messages.findLastIndex((message) => message.role === "user");
  const user = request.messages[userIndex];
  const prompt = user ? (getTextContent(user.content) ?? "") : "";
  const turn = request.messages.slice(userIndex + 1);
  if (/help me get ready for the aquarium trip/i.test(prompt))
    return demoTripResponse(request, turn);
  if (/explore exhibits/i.test(prompt)) return demoExhibitResponse(request, turn);
  if (/something hands.on/i.test(prompt)) return demoRefinementResponse(request, turn);
  if (/I choose [“"]?Review trip details/i.test(prompt))
    return demoSchoolChoiceResponse(request, turn, "review");
  if (/I choose [“"]?Complete permission slip/i.test(prompt))
    return demoSchoolChoiceResponse(request, turn, "complete");
  if (
    /I choose [“"]?(?:Rocky Shore|Kelp Forest|Open Sea)/i.test(prompt) ||
    /^(?:Rocky Shore|Kelp Forest|Open Sea)$/i.test(prompt)
  ) {
    const label = /Rocky Shore|Kelp Forest|Open Sea/i.exec(prompt)?.[0] ?? "that exhibit";
    return {
      content: `You chose ${label}. I can help plan a route through the aquarium or review the school-trip details next.`,
    };
  }
  if (/email|inbox/i.test(prompt)) return demoMailResponse(request, turn);
  const url = targetUrl(prompt);
  if (!url)
    return {
      content:
        "Try “Find cool stuff on Hacker News”, “Summarize copilotkit.ai”, “Check my emails for the school trip”, or “Research Monterey Bay Aquarium”.",
    };

  // Only the latest turn can satisfy this request; older browser reads cannot suppress a new visit.
  const calls = new Set(
    turn.flatMap((message) =>
      (message.tool_calls ?? [])
        .filter((call) => call.function.name === "browse_web")
        .map((call) => call.id),
    ),
  );
  const result = turn.findLast(
    (message) =>
      message.role === "tool" &&
      message.tool_call_id &&
      (calls.has(message.tool_call_id) ||
        message.tool_call_id.startsWith("call_openmuse_demo_browse_")),
  );
  if (result) return summarizePage(result);
  if (!request.tools?.some((tool) => tool.function.name === "browse_web"))
    return { content: "The browse_web tool is not available. Start the API with browser support." };
  return {
    content: url.includes("ycombinator")
      ? "I’ll open Hacker News and read the front page."
      : url.includes("montereybayaquarium")
        ? "I’ll research the exhibits on the aquarium’s own website."
        : "I’ll open CopilotKit and read the page.",
    toolCalls: [
      {
        id: `call_openmuse_demo_browse_${randomUUID()}`,
        name: "browse_web",
        arguments: JSON.stringify({ url }),
      },
    ],
  };
}

export function createDemoModel(
  options: { port?: number; latency?: number; firstByteDelay?: number } = {},
) {
  const latency = options.latency ?? 80;
  const firstByteDelay = options.firstByteDelay ?? options.latency ?? 1500;
  if (![latency, firstByteDelay].every((value) => Number.isFinite(value) && value >= 0))
    throw new Error("Demo model delays must be finite nonnegative milliseconds");
  return new LLMock({
    host: "127.0.0.1",
    port: options.port ?? 0,
    latency,
    chunkSize: 14,
    strict: true,
    logLevel: "silent",
    journalMaxEntries: 100,
  }).on({ model: "openmuse-browser-demo" }, demoResponse, {
    streamingProfile: { ttft: firstByteDelay },
  });
}