Multiplayer agent harness for work.
Serve textual files with an explicit UTF-8 charset (#381)
Regan Bellb545b8fa0763
diff --git a/src/api/http.ts b/src/api/http.tsindex 7a1bf5e0a..56f196e0f 100644--- a/src/api/http.ts+++ b/src/api/http.ts@@ -7,6 +7,43 @@ export function sendJson(res: ServerResponse, status: number, body: unknown): vores.end(data);}+export function contentTypeWithUtf8Charset(contentType: string): string {+ let parameterStart = -1;+ let inQuotes = false;+ let escaped = false;+ for (let i = 0; i < contentType.length; i += 1) {+ const character = contentType[i]!;+ if (escaped) {+ escaped = false;+ } else if (inQuotes && character === "\\") {+ escaped = true;+ } else if (character === '"') {+ inQuotes = !inQuotes;+ } else if (!inQuotes && character === ";") {+ if (parameterStart >= 0 && /^\s*charset\s*=/i.test(contentType.slice(parameterStart, i))) return contentType;+ parameterStart = i + 1;+ }+ }+ if (parameterStart >= 0 && /^\s*charset\s*=/i.test(contentType.slice(parameterStart))) return contentType;+ const mime = contentType.split(";", 1)[0]!.trim().toLowerCase();+ const textual =+ mime.startsWith("text/") ||+ mime === "application/json" ||+ mime === "application/xml" ||+ mime === "application/yaml" ||+ mime === "application/x-yaml" ||+ mime === "application/javascript" ||+ mime === "application/x-javascript" ||+ mime === "application/graphql" ||+ mime === "application/sql" ||+ mime === "application/toml" ||+ mime === "image/svg+xml" ||+ mime.endsWith("+json") ||+ mime.endsWith("+xml") ||+ mime.endsWith("+yaml");+ return textual ? `${contentType}; charset=utf-8` : contentType;+}+export function pipeToResponse(res: ServerResponse,stream: NodeJS.ReadableStream & { destroy?: () => void },diff --git a/src/api/routes/admin/files.ts b/src/api/routes/admin/files.tsindex 49af4e542..9e69cc6ad 100644--- a/src/api/routes/admin/files.ts+++ b/src/api/routes/admin/files.ts@@ -2,7 +2,7 @@ import { parseScopeId } from "../../../types.ts";import { ByteSourceTooLargeError } from "../../../files/durable-byte-store.ts";import { fileArtifactId } from "../../../files/file-artifact-store.ts";import { MAX_ATTACHMENT_BYTES, mimeFromName, safeAttachmentName } from "../../../core/attachments.ts";-import { contentDispositionAttachment, pipeToResponse, sendJson } from "../../http.ts";+import { contentDispositionAttachment, contentTypeWithUtf8Charset, pipeToResponse, sendJson } from "../../http.ts";import { audit, authorizeAdmin, requireScopedAdmin } from "../shared.ts";import { type ApiCtx } from "../route.ts";import { discoverScopes, FILES_PAGE_SIZE } from "./common.ts";@@ -68,7 +68,7 @@ export async function downloadAdminFile(ctx: ApiCtx): Promise<void> {const opened = await deps.files.open(id);if (!opened) return sendJson(res, 404, { error: "not_found" });res.writeHead(200, {- "content-type": inline ? mime : "application/octet-stream",+ "content-type": inline ? contentTypeWithUtf8Charset(art.mimetype) : "application/octet-stream","content-length": String(opened.sizeBytes),"content-disposition": contentDispositionAttachment(art.name, inline ? "inline" : "attachment"),"x-content-type-options": "nosniff",diff --git a/src/api/routes/surface.ts b/src/api/routes/surface.tsindex 3fbe270af..3b56d4fcc 100644--- a/src/api/routes/surface.ts+++ b/src/api/routes/surface.ts@@ -18,7 +18,7 @@ import { builtInModelCatalog, selectableCatalogForHarness, selectableModelCataloimport { errMessage } from "../../util/errors.ts";import { renderAgentApis } from "../agent-api-catalog.ts";import { mintCapabilityToken, CAPABILITY_TTL_MS } from "../../auth/capability-token.ts";-import { pipeToResponse, sendJson } from "../http.ts";+import { contentTypeWithUtf8Charset, pipeToResponse, sendJson } from "../http.ts";import { audit, isObj, orgScope } from "./shared.ts";import { type ApiCtx, type Route } from "./route.ts";import {@@ -252,7 +252,7 @@ async function getFileContent(ctx: ApiCtx): Promise<void> {const opened = await app.openFileForViewer(id, viewer);if (!opened) return sendJson(res, 404, { error: "not_found" });res.writeHead(200, {- "content-type": opened.mimetype || "application/octet-stream",+ "content-type": contentTypeWithUtf8Charset(opened.mimetype || "application/octet-stream"),"content-length": String(opened.sizeBytes),"content-disposition": `inline; filename*=UTF-8''${encodeURIComponent(opened.name)}`,});diff --git a/test/admin-observability.test.ts b/test/admin-observability.test.tsindex 529a498b0..f8db7c4c3 100644--- a/test/admin-observability.test.ts+++ b/test/admin-observability.test.ts@@ -84,7 +84,7 @@ test("an org admin sees conversations, transcripts, files, and runs top-down", aassert.equal(download.status, 200);assert.equal(download.headers.get("content-type"),- "text/plain",+ "text/plain; charset=utf-8","browser-renderable types open in the browser instead of downloading",);assert.match(download.headers.get("content-disposition") ?? "", /^inline/);diff --git a/test/agent-files-route.test.ts b/test/agent-files-route.test.tsindex a60fc371a..958fe5638 100644--- a/test/agent-files-route.test.ts+++ b/test/agent-files-route.test.ts@@ -6,6 +6,7 @@ import type { AddressInfo } from "node:net";import type { Server } from "node:http";import { buildApp, type BuiltApp } from "../src/wiring.ts";import { createServer } from "../src/api/server.ts";+import { contentTypeWithUtf8Charset } from "../src/api/http.ts";import { fileArtifactId } from "../src/files/file-artifact-store.ts";import { mintCapabilityToken, CAPABILITY_TTL_MS, CONTROL_PLANE_AUD } from "../src/auth/capability-token.ts";import { scopeId } from "../src/types.ts";@@ -13,12 +14,30 @@ import { testConfig } from "./support/test-config.ts";const SECRET = "agent-files-secret".repeat(3);+it("detects charset parameters outside quoted parameter values", () => {+ assert.equal(+ contentTypeWithUtf8Charset('text/plain; note="x;charset=bogus"'),+ 'text/plain; note="x;charset=bogus"; charset=utf-8',+ );+ assert.equal(+ contentTypeWithUtf8Charset("text/plain; format=flowed; charset=iso-8859-1"),+ "text/plain; format=flowed; charset=iso-8859-1",+ );+ assert.equal(+ contentTypeWithUtf8Charset("text/plain; format=flowed; ChArSeT = utf-16"),+ "text/plain; format=flowed; ChArSeT = utf-16",+ );+ assert.equal(contentTypeWithUtf8Charset('text/plain; note="one;two"'), 'text/plain; note="one;two"; charset=utf-8');+});+describe("agent files self-API", async () => {let server: Server;let base: string;let built: BuiltApp;let mineId: string;let theirsId: string;+ let encodedId: string;+ let binaryId: string;const capFor = (actorId: string) =>mintCapabilityToken(@@ -41,6 +60,8 @@ describe("agent files self-API", async () => {base = `http://localhost:${(server.address() as AddressInfo).port}`;mineId = fileArtifactId("mine", "out", 0);theirsId = fileArtifactId("theirs", "out", 0);+ encodedId = fileArtifactId("encoded", "out", 0);+ binaryId = fileArtifactId("binary", "out", 0);await built.files.put({id: mineId,ownerScopeId: scopeId("personal", "U1"),@@ -61,6 +82,26 @@ describe("agent files self-API", async () => {data: Buffer.from("theirs"),direction: "out",});+ await built.files.put({+ id: encodedId,+ ownerScopeId: scopeId("personal", "U1"),+ createdBy: "U1",+ name: "encoded.json",+ path: `artifacts/${encodedId}/encoded.json`,+ mimetype: "application/json; charset=iso-8859-1",+ data: Buffer.from("{}"),+ direction: "out",+ });+ await built.files.put({+ id: binaryId,+ ownerScopeId: scopeId("personal", "U1"),+ createdBy: "U1",+ name: "binary.png",+ path: `artifacts/${binaryId}/binary.png`,+ mimetype: "image/png",+ data: Buffer.from([0x89, 0x50, 0x4e, 0x47]),+ direction: "out",+ });});after(async () => {@@ -75,10 +116,7 @@ describe("agent files self-API", async () => {const res = await get("/v1/files", await capFor("U1"));assert.equal(res.status, 200);const page = (await res.json()) as { owned: Array<{ id: string }>; shared: Array<{ id: string }> };- assert.deepEqual(- page.owned.map((file) => file.id),- [mineId],- );+ assert.deepEqual(page.owned.map((file) => file.id).sort(), [mineId, encodedId, binaryId].sort());assert.deepEqual(page.shared, []);});@@ -86,4 +124,16 @@ describe("agent files self-API", async () => {const res = await get(`/v1/files/${theirsId}/content`, await capFor("U1"));assert.equal(res.status, 404);});++ it("adds UTF-8 to textual file content without changing explicit charsets or binary types", async () => {+ const token = await capFor("U1");+ const text = await get(`/v1/files/${mineId}/content`, token);+ const encoded = await get(`/v1/files/${encodedId}/content`, token);+ const binary = await get(`/v1/files/${binaryId}/content`, token);++ assert.equal(text.headers.get("content-type"), "text/plain; charset=utf-8");+ assert.equal(encoded.headers.get("content-type"), "application/json; charset=iso-8859-1");+ assert.equal(binary.headers.get("content-type"), "image/png");+ await Promise.all([text.arrayBuffer(), encoded.arrayBuffer(), binary.arrayBuffer()]);+ });});