TypeScript
fix(server): load PR uploads from SSO organizations (#21632)
Julius Marmingea09aef3287d2
diff --git a/apps/server/src/assets/AssetAccess.test.ts b/apps/server/src/assets/AssetAccess.test.tsindex b983a431d79..d762a15a830 100644--- a/apps/server/src/assets/AssetAccess.test.ts+++ b/apps/server/src/assets/AssetAccess.test.ts@@ -35,6 +35,7 @@ import * as WorkspacePaths from "../workspace/WorkspacePaths.ts";import { assetFileResponse } from "../http.ts";import { ASSET_ROUTE_PREFIX, issueAssetUrl, readMcpAppHtml, resolveAsset } from "./AssetAccess.ts";import * as NativeAppIconResolver from "./NativeAppIconResolver.ts";+import * as RepositoryIdentityResolver from "../project/RepositoryIdentityResolver.ts";import { symlinksSupported } from "@t3tools/shared/testing/symlinks";import { fakeSourceControlDriverInstance } from "@t3tools/source-control-testing/fakeSourceControlDriver";import * as BuiltInDrivers from "../sourceControl/builtInDrivers.ts";@@ -1497,7 +1498,11 @@ describe("AssetAccess", () => {);it.effect("serves a host's credentialed media through that host's driver", () => {- const fetched: Array<{ url: string; headers: Readonly<Record<string, string>> }> = [];+ const fetched: Array<{+ url: string;+ headers: Readonly<Record<string, string>>;+ repository?: string;+ }> = [];const url = "https://github.com/owner/repo/blob/main/docs/shot.png";const media = sourceControlDefinitions.mediaUrl(url)!;const layerDrivers = Layer.succeed(BuiltInDrivers.SourceControlDrivers, [@@ -1516,6 +1521,17 @@ describe("AssetAccess", () => {}),},]);+ // Clients that predate `repository` get the checkout's.+ const layerIdentities = Layer.mock(RepositoryIdentityResolver.RepositoryIdentityResolver)({+ resolve: () =>+ Effect.succeed({+ canonicalKey: "github.com/owner/repo",+ locator: { source: "git-remote", remoteName: "origin", remoteUrl: "" },+ provider: media.definition.kind,+ owner: "owner",+ name: "repo",+ }),+ });return Effect.gen(function* () {const decodeAssetCreateUrlInput = Schema.decodeEffect(AssetCreateUrlInput);@@ -1539,12 +1555,21 @@ describe("AssetAccess", () => {range: "bytes=0-4",cookie: "session",});- expect(fetched).toEqual([{ url: media.fetchUrl, headers: { range: "bytes=0-4" } }]);+ expect(fetched).toEqual([+ { url: media.fetchUrl, headers: { range: "bytes=0-4" }, repository: "owner/repo" },+ ]);expect(response.status).toBe(206);// The upstream sent no type, so it comes from the file name the host gave the media.expect(response.headers["content-type"]).toBe("image/png");expect(response.headers["content-length"]).toBe("5");- }).pipe(Effect.provide(Layer.merge(layerTest, layerDrivers)), Effect.scoped);++ // A pull request's own repository wins over the checkout it is shown in.+ yield* sourceControlMediaResponse({ ...asset, repository: "enterprise/private" }, {});+ expect(fetched.at(-1)?.repository).toBe("enterprise/private");+ }).pipe(+ Effect.provide(Layer.mergeAll(layerTest, layerDrivers, layerIdentities)),+ Effect.scoped,+ );});it.effect("refuses media no host keeps behind a credential, without echoing the URL", () =>diff --git a/apps/server/src/assets/sourceControlMediaResponse.ts b/apps/server/src/assets/sourceControlMediaResponse.tsindex 96e96f32a17..0cc653bc061 100644--- a/apps/server/src/assets/sourceControlMediaResponse.ts+++ b/apps/server/src/assets/sourceControlMediaResponse.ts@@ -4,6 +4,7 @@ import * as Effect from "effect/Effect";import * as Option from "effect/Option";import { HttpServerResponse } from "effect/http";+import * as RepositoryIdentityResolver from "../project/RepositoryIdentityResolver.ts";import * as BuiltInDrivers from "../sourceControl/builtInDrivers.ts";import { sourceControlDefinitions } from "../sourceControl/sourceControlDefinitions.ts";@@ -25,7 +26,12 @@ const SVG_CONTENT_SECURITY_POLICY = "default-src 'none'; style-src 'unsafe-inlinexport const sourceControlMediaResponse = Effect.fn("SourceControlMediaResponse.sourceControlMediaResponse",)(function* (- asset: { readonly url: string; readonly cwd: string; readonly expiresAt: number },+ asset: {+ readonly url: string;+ readonly cwd: string;+ readonly repository?: string;+ readonly expiresAt: number;+ },requestHeaders: Record<string, string | undefined>,) {const remainingSeconds = Math.floor((asset.expiresAt - (yield* Clock.currentTimeMillis)) / 1000);@@ -46,7 +52,20 @@ export const sourceControlMediaResponse = Effect.fn(const value = requestHeaders[name];if (value !== undefined) forwarded[name] = value;}- const response = yield* fetchMedia({ url: asset.url, headers: forwarded });+ // Clients that predate `repository` fall back to the repository checked out at `cwd`.+ let repository = asset.repository;+ if (repository === undefined) {+ const identities = yield* RepositoryIdentityResolver.RepositoryIdentityResolver;+ const identity = yield* identities.resolve(asset.cwd);+ if (identity?.provider === media.definition.kind && identity.owner && identity.name) {+ repository = `${identity.owner}/${identity.name}`;+ }+ }+ const response = yield* fetchMedia({+ url: asset.url,+ headers: forwarded,+ ...(repository === undefined ? {} : { repository }),+ });if (response === null) return HttpServerResponse.empty({ status: 502, headers });// It carries none of the upstream entity headers: a `content-length` with no body behind it// holds the connection open until the browser gives up on it.diff --git a/packages/source-control-core/src/server/driver.ts b/packages/source-control-core/src/server/driver.tsindex 7226faad651..c4fb4106a20 100644--- a/packages/source-control-core/src/server/driver.ts+++ b/packages/source-control-core/src/server/driver.ts@@ -25,6 +25,8 @@ export interface SourceControlMediaRequest {readonly url: string;/** Request headers to forward, such as `range`. */readonly headers: Readonly<Record<string, string>>;+ /** `owner/name` of the change request the media is in, else of the checkout it was shown in. */+ readonly repository?: string;}/** The server side of a definition's `credentials.appSignIn`. */diff --git a/packages/source-control-github/src/server/gitHubMediaFetch.test.ts b/packages/source-control-github/src/server/gitHubMediaFetch.test.tsindex 07e2bbd8f72..8248a76e4b0 100644--- a/packages/source-control-github/src/server/gitHubMediaFetch.test.ts+++ b/packages/source-control-github/src/server/gitHubMediaFetch.test.ts@@ -2,10 +2,78 @@ import { describe, expect, it } from "@effect/vitest";import * as Effect from "effect/Effect";import * as Layer from "effect/Layer";import * as Redacted from "effect/Redacted";+import * as TestClock from "effect/testing/TestClock";import { HttpClient, HttpClientResponse } from "effect/http";import * as GitHubCredentials from "./GitHubCredentials.ts";import * as GitHubMediaFetch from "./gitHubMediaFetch.ts";+import * as GitHubRest from "./GitHubRest.ts";++const PR_REPOSITORY = "enterprise/private";++/** An SSO organization: the upload URL answers with a sign-in page, a rendered signed URL with bytes. */+const makeSsoFixture = (uploadUrl: string) => {+ const state = { account: "account-a", expired: new Set<string>() };+ const rendered: Array<GitHubRest.GitHubRestInput> = [];+ const signedUrls: Array<string> = [];+ const requests: Array<{ url: string; authorization?: string; range?: string }> = [];+ const credential = (host: string) =>+ Effect.sync(() => ({+ host,+ token: Redacted.make(state.account),+ fingerprint: state.account,+ source: "gh" as const,+ }));+ const layer = Layer.mergeAll(+ Layer.mock(GitHubCredentials.GitHubCredentials)({ get: credential }),+ Layer.mock(GitHubRest.GitHubRest)({+ rest: (input) =>+ Effect.sync(() => {+ rendered.push(input);+ const signed = `https://private-user-images.githubusercontent.com/1/upload.mp4?jwt=${state.account}-${rendered.length}&x=1`;+ const inRepository =+ (input.body as { readonly context?: string } | undefined)?.context === PR_REPOSITORY;+ if (inRepository) signedUrls.push(signed);+ return {+ status: 200,+ headers: {},+ body: `<p><img src="${inRepository ? signed.replaceAll("&", "&") : uploadUrl}" alt=""></p>`,+ truncated: false,+ invalidUtf8: false,+ };+ }),+ }),+ Layer.succeed(+ HttpClient.HttpClient,+ HttpClient.make((request) =>+ Effect.sync(() => {+ requests.push({+ url: request.url,+ ...(request.headers.authorization+ ? { authorization: request.headers.authorization }+ : {}),+ ...(request.headers.range ? { range: request.headers.range } : {}),+ });+ if (request.url === uploadUrl) {+ return HttpClientResponse.fromWeb(+ request,+ new Response("<html>Sign in with SSO</html>", {+ headers: { "content-type": "text/html; charset=utf-8" },+ }),+ );+ }+ expect(signedUrls).toContain(request.url);+ const status = state.expired.has(request.url) ? 403 : request.headers.range ? 206 : 200;+ return HttpClientResponse.fromWeb(+ request,+ new Response(null, { status, headers: { "content-type": "video/mp4" } }),+ );+ }),+ ),+ ),+ );+ return { layer, state, rendered, signedUrls, requests };+};describe("GitHubMediaFetch", () => {it.effect("loads private media immediately after login with the GitHub credential", () => {@@ -24,17 +92,20 @@ describe("GitHubMediaFetch", () => {expect(authorizations).toEqual([undefined, "Bearer signed-in", "Bearer signed-in"]);}).pipe(Effect.provide(- Layer.mock(GitHubCredentials.GitHubCredentials)({- get: (host) =>- ++lookups === 1- ? Effect.fail(new GitHubCredentials.GitHubNotSignedInError({ host }))- : Effect.succeed({- host,- token: Redacted.make("signed-in"),- source: "gh" as const,- fingerprint: "fingerprint",- }),- }),+ Layer.merge(+ Layer.mock(GitHubRest.GitHubRest)({}),+ Layer.mock(GitHubCredentials.GitHubCredentials)({+ get: (host) =>+ ++lookups === 1+ ? Effect.fail(new GitHubCredentials.GitHubNotSignedInError({ host }))+ : Effect.succeed({+ host,+ token: Redacted.make("signed-in"),+ source: "gh" as const,+ fingerprint: "fingerprint",+ }),+ }),+ ),),Effect.provideService(HttpClient.HttpClient,@@ -70,15 +141,18 @@ describe("GitHubMediaFetch", () => {]);}).pipe(Effect.provide(- Layer.mock(GitHubCredentials.GitHubCredentials)({- get: (host) =>- Effect.succeed({- host,- token: Redacted.make("signed-in"),- source: "gh" as const,- fingerprint: "fingerprint",- }),- }),+ Layer.merge(+ Layer.mock(GitHubRest.GitHubRest)({}),+ Layer.mock(GitHubCredentials.GitHubCredentials)({+ get: (host) =>+ Effect.succeed({+ host,+ token: Redacted.make("signed-in"),+ source: "gh" as const,+ fingerprint: "fingerprint",+ }),+ }),+ ),),Effect.provideService(HttpClient.HttpClient,@@ -101,4 +175,68 @@ describe("GitHubMediaFetch", () => {Effect.scoped,);});++ it.effect(+ "renders SSO uploads in the PR repository and reuses the signed URL for a range",+ () => {+ const url = "https://github.com/user-attachments/assets/explicit-pr-context";+ const fixture = makeSsoFixture(url);+ return Effect.gen(function* () {+ const fetchMedia = yield* GitHubMediaFetch.make;+ const request = { url, headers: {}, repository: PR_REPOSITORY };+ expect((yield* fetchMedia(request))?.status).toBe(200);+ expect((yield* fetchMedia({ ...request, headers: { range: "bytes=2-5" } }))?.status).toBe(+ 206,+ );+ expect(fixture.rendered).toHaveLength(1);+ expect(fixture.rendered[0]).toMatchObject({+ host: "github.com",+ method: "POST",+ path: "markdown",+ accept: "text/html",+ body: { text: ``, mode: "gfm", context: PR_REPOSITORY },+ });+ expect(fixture.requests).toEqual([+ { url, authorization: "Bearer account-a" },+ { url: fixture.signedUrls[0] },+ { url: fixture.signedUrls[0], range: "bytes=2-5" },+ ]);+ }).pipe(Effect.provide(fixture.layer), Effect.scoped);+ },+ );++ it.effect("renders a legacy upload in the repository its path names", () => {+ const url = `https://github.com/${PR_REPOSITORY}/assets/123/legacy`;+ const fixture = makeSsoFixture(url);+ return Effect.gen(function* () {+ const fetchMedia = yield* GitHubMediaFetch.make;+ const response = yield* fetchMedia({ url, headers: {}, repository: "personal/unrelated" });+ expect(response?.status).toBe(200);+ expect(fixture.rendered.map((input) => input.body)).toEqual([+ { text: ``, mode: "gfm", context: PR_REPOSITORY },+ ]);+ }).pipe(Effect.provide(fixture.layer), Effect.scoped);+ });++ it.effect("renders a fresh signed URL after an account switch, early expiry, or its TTL", () => {+ const url = "https://github.com/user-attachments/assets/cache-lifecycle";+ const fixture = makeSsoFixture(url);+ const request = { url, headers: {}, repository: PR_REPOSITORY };+ return Effect.gen(function* () {+ const fetchMedia = yield* GitHubMediaFetch.make;+ yield* fetchMedia(request);+ fixture.state.account = "account-b";+ yield* fetchMedia(request);+ expect(fixture.rendered).toHaveLength(2);++ fixture.state.expired.add(fixture.signedUrls.at(-1)!);+ expect((yield* fetchMedia(request))?.status).toBe(200);+ expect(fixture.rendered).toHaveLength(3);++ yield* TestClock.adjust("6 minutes");+ yield* fetchMedia(request);+ expect(fixture.rendered).toHaveLength(4);+ expect(fixture.requests.at(-1)?.url).toBe(fixture.signedUrls.at(-1));+ }).pipe(Effect.provide(fixture.layer), Effect.scoped);+ });});diff --git a/packages/source-control-github/src/server/gitHubMediaFetch.ts b/packages/source-control-github/src/server/gitHubMediaFetch.tsindex 4dfb2a23e63..b03c0d327a6 100644--- a/packages/source-control-github/src/server/gitHubMediaFetch.ts+++ b/packages/source-control-github/src/server/gitHubMediaFetch.ts@@ -1,4 +1,5 @@import type { SourceControlMediaRequest } from "@t3tools/source-control-core/server/driver";+import * as Clock from "effect/Clock";import * as Effect from "effect/Effect";import * as Redacted from "effect/Redacted";import {@@ -11,6 +12,7 @@ import {import { GITHUB_CREDENTIALED_MEDIA_HOSTS } from "../shared/gitHubMedia.ts";import * as GitHubCredentials from "./GitHubCredentials.ts";+import * as GitHubRest from "./GitHubRest.ts";/*** Everything a redirect leads to — the presigned object stores GitHub hands assets off to —@@ -18,51 +20,170 @@ import * as GitHubCredentials from "./GitHubCredentials.ts";* bearer token.*/const isCredentialedHost = (url: string) => {- try {- return GITHUB_CREDENTIALED_MEDIA_HOSTS.has(new URL(url).hostname.toLowerCase());- } catch {- return false;- }+ const target = URL.parse(url);+ return (+ target !== null &&+ target.protocol === "https:" &&+ target.port === "" &&+ GITHUB_CREDENTIALED_MEDIA_HOSTS.has(target.hostname.toLowerCase())+ );};const MAX_REDIRECTS = 3;const MANUAL_REDIRECT: RequestInit = { redirect: "manual" };+const MEDIA_CONTENT_TYPE_PATTERN = /^(?:image|video|audio)\/[\w!#$&^.+-]+$/i;+const ATTACHMENT_PATH_PATTERN =+ /^\/(?:user-attachments\/assets\/[\w-]+|[^/]+\/[^/]+\/assets\/\d+\/[\w-]+)$/u;+const LEGACY_ATTACHMENT_REPOSITORY_PATTERN = /^\/([^/]+\/[^/]+)\/assets\//u;+const REPOSITORY_PATTERN = /^[\w.-]+\/[\w.-]+$/u;+// GitHub's signed upload links last about five minutes.+const SIGNED_URL_CACHE_TTL_MS = 4 * 60_000;+const SIGNED_URL_CACHE_MAX_ENTRIES = 64;++const contentTypeOf = (response: HttpClientResponse.HttpClientResponse) =>+ response.headers["content-type"]?.split(";", 1)[0]?.trim().toLowerCase() ?? "";++/** The repository an upload has to be rendered in, or `undefined` when it is not an upload. */+const uploadRepository = (request: SourceControlMediaRequest) => {+ const target = URL.parse(request.url);+ if (+ target === null ||+ target.protocol !== "https:" ||+ target.hostname !== "github.com" ||+ target.port !== "" ||+ target.username !== "" ||+ target.password !== "" ||+ !ATTACHMENT_PATH_PATTERN.test(target.pathname)+ ) {+ return undefined;+ }+ // A legacy upload names its repository; one the caller supplies cannot override that.+ const repository =+ LEGACY_ATTACHMENT_REPOSITORY_PATTERN.exec(target.pathname)?.[1] ?? request.repository;+ // An unusable explicit context must not fall back to an unrelated checkout.+ return repository !== undefined && REPOSITORY_PATTERN.test(repository) ? repository : undefined;+};export const make = Effect.gen(function* () {const credentials = yield* GitHubCredentials.GitHubCredentials;+ const rest = yield* GitHubRest.GitHubRest;const httpClient = HttpClient.withScope(yield* HttpClient.HttpClient);+ const signedUrls = new Map<string, { readonly at: number; readonly url: string }>();// Both media hosts are served by github.com's account, which is the host `gh` stores it under.// Read on every fetch, so media loads as soon as the user signs in.- const token = Effect.suspend(() => credentials.get("github.com")).pipe(- Effect.map((credential) => credential.token),+ const credential = Effect.suspend(() => credentials.get("github.com")).pipe(Effect.orElseSucceed(() => null),);- return Effect.fn("GitHubMediaFetch.fetchMedia")(function* (request: SourceControlMediaRequest) {- const bearer = yield* token;- let target = request.url;+ const fetchFollowingRedirects = Effect.fnUntraced(function* (+ url: string,+ headers: Readonly<Record<string, string>>,+ token: Redacted.Redacted<string> | null,+ ) {+ let target = url;for (let hop = 0; ; hop += 1) {const authorization =- bearer !== null && isCredentialedHost(target) ? `Bearer ${Redacted.value(bearer)}` : null;+ token !== null && isCredentialedHost(target) ? `Bearer ${Redacted.value(token)}` : null;const response: HttpClientResponse.HttpClientResponse = yield* httpClient.execute(HttpClientRequest.get(target).pipe(HttpClientRequest.setHeaders({- ...request.headers,+ ...headers,// The bytes are streamed straight through, so never let an encoding layer in."accept-encoding": "identity",...(authorization === null ? {} : { authorization }),}),),)- .pipe(Effect.provideService(FetchHttpClient.RequestInit, MANUAL_REDIRECT));+ .pipe(+ Effect.provideService(FetchHttpClient.RequestInit, MANUAL_REDIRECT),+ // A signed URL's query string is a bearer capability; keep it out of tracing spans.+ Effect.provideService(HttpClient.TracerDisabledWhen, () => true),+ );const location = response.headers.location;if (response.status < 300 || response.status >= 400) return response;if (!location || hop >= MAX_REDIRECTS) return null;- const next = new URL(location, target);- if (next.protocol !== "https:") return null;+ const next = URL.parse(location, target);+ if (next === null || next.protocol !== "https:" || next.username || next.password) {+ return null;+ }target = next.toString();}});++ // An organization with SAML SSO answers a token-authenticated upload request with its sign-in+ // page. Rendering the link as markdown in the repository's context hands back a signed URL.+ const renderSignedUrl = Effect.fnUntraced(function* (url: string, repository: string) {+ const rendered = yield* rest+ .rest({+ host: "github.com",+ operation: "renderMarkdown",+ method: "POST",+ path: "markdown",+ accept: "text/html",+ body: { text: ``, mode: "gfm", context: repository },+ })+ .pipe(Effect.orElseSucceed(() => null));+ if (rendered === null || rendered.status !== 200 || rendered.truncated) return null;+ for (const match of rendered.body.matchAll(/<img\b[^>]*\ssrc\s*=\s*(["'])(.*?)\1/giu)) {+ const signed = URL.parse(match[2]?.replaceAll("&", "&") ?? "");+ if (+ signed?.protocol === "https:" &&+ signed.hostname === "private-user-images.githubusercontent.com" &&+ signed.port === "" &&+ signed.username === "" &&+ signed.password === "" &&+ signed.searchParams.get("jwt")+ ) {+ return signed.toString();+ }+ }+ return null;+ });++ return Effect.fn("GitHubMediaFetch.fetchMedia")(function* (request: SourceControlMediaRequest) {+ const signedIn = yield* credential;+ const repository = signedIn === null ? undefined : uploadRepository(request);+ const cacheKey =+ signedIn === null || repository === undefined+ ? null+ : [signedIn.fingerprint, repository, request.url].map(encodeURIComponent).join("|");+ const now = yield* Clock.currentTimeMillis;+ const cached = cacheKey === null ? undefined : signedUrls.get(cacheKey);+ let response: HttpClientResponse.HttpClientResponse | null;+ if (cacheKey !== null && cached !== undefined && now - cached.at < SIGNED_URL_CACHE_TTL_MS) {+ response = yield* fetchFollowingRedirects(cached.url, request.headers, null);+ if (response === null || response.status >= 400) signedUrls.delete(cacheKey);+ // A signed URL can expire before its TTL; only then is a fresh one worth rendering.+ if (response?.status !== 403) return response;+ } else {+ if (cacheKey !== null) signedUrls.delete(cacheKey);+ response = yield* fetchFollowingRedirects(+ request.url,+ request.headers,+ signedIn?.token ?? null,+ );+ if (response === null || contentTypeOf(response) !== "text/html") return response;+ }+ if (cacheKey === null || repository === undefined || signedIn === null) return response;+ const signed = yield* renderSignedUrl(request.url, repository);+ if (signed === null) return response;+ const media = yield* fetchFollowingRedirects(signed, request.headers, null);+ if (+ (media?.status === 200 || media?.status === 206) &&+ MEDIA_CONTENT_TYPE_PATTERN.test(contentTypeOf(media))+ ) {+ // The render can run under another account if the user switched mid-request.+ const current = yield* credential;+ if (current?.fingerprint === signedIn.fingerprint) {+ if (signedUrls.size >= SIGNED_URL_CACHE_MAX_ENTRIES) {+ const oldest = signedUrls.keys().next().value;+ if (oldest !== undefined) signedUrls.delete(oldest);+ }+ signedUrls.set(cacheKey, { at: now, url: signed });+ }+ }+ return media;+ });});