Skip to content
TypeScript

fix(server): load PR uploads from SSO organizations (#21632)

Julius Marmingea09aef3287d2
diff --git a/apps/server/src/assets/AssetAccess.test.ts b/apps/server/src/assets/AssetAccess.test.ts
index b983a431d79..d762a15a830 100644
--- a/apps/server/src/assets/AssetAccess.test.ts
+++ b/apps/server/src/assets/AssetAccess.test.ts
@@ -35,6 +35,7 @@ import * as WorkspacePaths from "../workspace/WorkspacePaths.ts";
import { assetFileResponse } from "../http.ts";
import { ASSET_ROUTE_PREFIX, issueAssetUrl, readMcpAppHtml, resolveAsset } from "./AssetAccess.ts";
import * as NativeAppIconResolver from "./NativeAppIconResolver.ts";
+import * as RepositoryIdentityResolver from "../project/RepositoryIdentityResolver.ts";
import { symlinksSupported } from "@t3tools/shared/testing/symlinks";
import { fakeSourceControlDriverInstance } from "@t3tools/source-control-testing/fakeSourceControlDriver";
import * as BuiltInDrivers from "../sourceControl/builtInDrivers.ts";
@@ -1497,7 +1498,11 @@ describe("AssetAccess", () => {
);
it.effect("serves a host's credentialed media through that host's driver", () => {
- const fetched: Array<{ url: string; headers: Readonly<Record<string, string>> }> = [];
+ const fetched: Array<{
+ url: string;
+ headers: Readonly<Record<string, string>>;
+ repository?: string;
+ }> = [];
const url = "https://github.com/owner/repo/blob/main/docs/shot.png";
const media = sourceControlDefinitions.mediaUrl(url)!;
const layerDrivers = Layer.succeed(BuiltInDrivers.SourceControlDrivers, [
@@ -1516,6 +1521,17 @@ describe("AssetAccess", () => {
}),
},
]);
+ // Clients that predate `repository` get the checkout's.
+ const layerIdentities = Layer.mock(RepositoryIdentityResolver.RepositoryIdentityResolver)({
+ resolve: () =>
+ Effect.succeed({
+ canonicalKey: "github.com/owner/repo",
+ locator: { source: "git-remote", remoteName: "origin", remoteUrl: "" },
+ provider: media.definition.kind,
+ owner: "owner",
+ name: "repo",
+ }),
+ });
return Effect.gen(function* () {
const decodeAssetCreateUrlInput = Schema.decodeEffect(AssetCreateUrlInput);
@@ -1539,12 +1555,21 @@ describe("AssetAccess", () => {
range: "bytes=0-4",
cookie: "session",
});
- expect(fetched).toEqual([{ url: media.fetchUrl, headers: { range: "bytes=0-4" } }]);
+ expect(fetched).toEqual([
+ { url: media.fetchUrl, headers: { range: "bytes=0-4" }, repository: "owner/repo" },
+ ]);
expect(response.status).toBe(206);
// The upstream sent no type, so it comes from the file name the host gave the media.
expect(response.headers["content-type"]).toBe("image/png");
expect(response.headers["content-length"]).toBe("5");
- }).pipe(Effect.provide(Layer.merge(layerTest, layerDrivers)), Effect.scoped);
+
+ // A pull request's own repository wins over the checkout it is shown in.
+ yield* sourceControlMediaResponse({ ...asset, repository: "enterprise/private" }, {});
+ expect(fetched.at(-1)?.repository).toBe("enterprise/private");
+ }).pipe(
+ Effect.provide(Layer.mergeAll(layerTest, layerDrivers, layerIdentities)),
+ Effect.scoped,
+ );
});
it.effect("refuses media no host keeps behind a credential, without echoing the URL", () =>
diff --git a/apps/server/src/assets/sourceControlMediaResponse.ts b/apps/server/src/assets/sourceControlMediaResponse.ts
index 96e96f32a17..0cc653bc061 100644
--- a/apps/server/src/assets/sourceControlMediaResponse.ts
+++ b/apps/server/src/assets/sourceControlMediaResponse.ts
@@ -4,6 +4,7 @@ import * as Effect from "effect/Effect";
import * as Option from "effect/Option";
import { HttpServerResponse } from "effect/http";
+import * as RepositoryIdentityResolver from "../project/RepositoryIdentityResolver.ts";
import * as BuiltInDrivers from "../sourceControl/builtInDrivers.ts";
import { sourceControlDefinitions } from "../sourceControl/sourceControlDefinitions.ts";
@@ -25,7 +26,12 @@ const SVG_CONTENT_SECURITY_POLICY = "default-src 'none'; style-src 'unsafe-inlin
export const sourceControlMediaResponse = Effect.fn(
"SourceControlMediaResponse.sourceControlMediaResponse",
)(function* (
- asset: { readonly url: string; readonly cwd: string; readonly expiresAt: number },
+ asset: {
+ readonly url: string;
+ readonly cwd: string;
+ readonly repository?: string;
+ readonly expiresAt: number;
+ },
requestHeaders: Record<string, string | undefined>,
) {
const remainingSeconds = Math.floor((asset.expiresAt - (yield* Clock.currentTimeMillis)) / 1000);
@@ -46,7 +52,20 @@ export const sourceControlMediaResponse = Effect.fn(
const value = requestHeaders[name];
if (value !== undefined) forwarded[name] = value;
}
- const response = yield* fetchMedia({ url: asset.url, headers: forwarded });
+ // Clients that predate `repository` fall back to the repository checked out at `cwd`.
+ let repository = asset.repository;
+ if (repository === undefined) {
+ const identities = yield* RepositoryIdentityResolver.RepositoryIdentityResolver;
+ const identity = yield* identities.resolve(asset.cwd);
+ if (identity?.provider === media.definition.kind && identity.owner && identity.name) {
+ repository = `${identity.owner}/${identity.name}`;
+ }
+ }
+ const response = yield* fetchMedia({
+ url: asset.url,
+ headers: forwarded,
+ ...(repository === undefined ? {} : { repository }),
+ });
if (response === null) return HttpServerResponse.empty({ status: 502, headers });
// It carries none of the upstream entity headers: a `content-length` with no body behind it
// holds the connection open until the browser gives up on it.
diff --git a/packages/source-control-core/src/server/driver.ts b/packages/source-control-core/src/server/driver.ts
index 7226faad651..c4fb4106a20 100644
--- a/packages/source-control-core/src/server/driver.ts
+++ b/packages/source-control-core/src/server/driver.ts
@@ -25,6 +25,8 @@ export interface SourceControlMediaRequest {
readonly url: string;
/** Request headers to forward, such as `range`. */
readonly headers: Readonly<Record<string, string>>;
+ /** `owner/name` of the change request the media is in, else of the checkout it was shown in. */
+ readonly repository?: string;
}
/** The server side of a definition's `credentials.appSignIn`. */
diff --git a/packages/source-control-github/src/server/gitHubMediaFetch.test.ts b/packages/source-control-github/src/server/gitHubMediaFetch.test.ts
index 07e2bbd8f72..8248a76e4b0 100644
--- a/packages/source-control-github/src/server/gitHubMediaFetch.test.ts
+++ b/packages/source-control-github/src/server/gitHubMediaFetch.test.ts
@@ -2,10 +2,78 @@ import { describe, expect, it } from "@effect/vitest";
import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";
import * as Redacted from "effect/Redacted";
+import * as TestClock from "effect/testing/TestClock";
import { HttpClient, HttpClientResponse } from "effect/http";
import * as GitHubCredentials from "./GitHubCredentials.ts";
import * as GitHubMediaFetch from "./gitHubMediaFetch.ts";
+import * as GitHubRest from "./GitHubRest.ts";
+
+const PR_REPOSITORY = "enterprise/private";
+
+/** An SSO organization: the upload URL answers with a sign-in page, a rendered signed URL with bytes. */
+const makeSsoFixture = (uploadUrl: string) => {
+ const state = { account: "account-a", expired: new Set<string>() };
+ const rendered: Array<GitHubRest.GitHubRestInput> = [];
+ const signedUrls: Array<string> = [];
+ const requests: Array<{ url: string; authorization?: string; range?: string }> = [];
+ const credential = (host: string) =>
+ Effect.sync(() => ({
+ host,
+ token: Redacted.make(state.account),
+ fingerprint: state.account,
+ source: "gh" as const,
+ }));
+ const layer = Layer.mergeAll(
+ Layer.mock(GitHubCredentials.GitHubCredentials)({ get: credential }),
+ Layer.mock(GitHubRest.GitHubRest)({
+ rest: (input) =>
+ Effect.sync(() => {
+ rendered.push(input);
+ const signed = `https://private-user-images.githubusercontent.com/1/upload.mp4?jwt=${state.account}-${rendered.length}&x=1`;
+ const inRepository =
+ (input.body as { readonly context?: string } | undefined)?.context === PR_REPOSITORY;
+ if (inRepository) signedUrls.push(signed);
+ return {
+ status: 200,
+ headers: {},
+ body: `<p><img src="${inRepository ? signed.replaceAll("&", "&amp;") : uploadUrl}" alt=""></p>`,
+ truncated: false,
+ invalidUtf8: false,
+ };
+ }),
+ }),
+ Layer.succeed(
+ HttpClient.HttpClient,
+ HttpClient.make((request) =>
+ Effect.sync(() => {
+ requests.push({
+ url: request.url,
+ ...(request.headers.authorization
+ ? { authorization: request.headers.authorization }
+ : {}),
+ ...(request.headers.range ? { range: request.headers.range } : {}),
+ });
+ if (request.url === uploadUrl) {
+ return HttpClientResponse.fromWeb(
+ request,
+ new Response("<html>Sign in with SSO</html>", {
+ headers: { "content-type": "text/html; charset=utf-8" },
+ }),
+ );
+ }
+ expect(signedUrls).toContain(request.url);
+ const status = state.expired.has(request.url) ? 403 : request.headers.range ? 206 : 200;
+ return HttpClientResponse.fromWeb(
+ request,
+ new Response(null, { status, headers: { "content-type": "video/mp4" } }),
+ );
+ }),
+ ),
+ ),
+ );
+ return { layer, state, rendered, signedUrls, requests };
+};
describe("GitHubMediaFetch", () => {
it.effect("loads private media immediately after login with the GitHub credential", () => {
@@ -24,17 +92,20 @@ describe("GitHubMediaFetch", () => {
expect(authorizations).toEqual([undefined, "Bearer signed-in", "Bearer signed-in"]);
}).pipe(
Effect.provide(
- Layer.mock(GitHubCredentials.GitHubCredentials)({
- get: (host) =>
- ++lookups === 1
- ? Effect.fail(new GitHubCredentials.GitHubNotSignedInError({ host }))
- : Effect.succeed({
- host,
- token: Redacted.make("signed-in"),
- source: "gh" as const,
- fingerprint: "fingerprint",
- }),
- }),
+ Layer.merge(
+ Layer.mock(GitHubRest.GitHubRest)({}),
+ Layer.mock(GitHubCredentials.GitHubCredentials)({
+ get: (host) =>
+ ++lookups === 1
+ ? Effect.fail(new GitHubCredentials.GitHubNotSignedInError({ host }))
+ : Effect.succeed({
+ host,
+ token: Redacted.make("signed-in"),
+ source: "gh" as const,
+ fingerprint: "fingerprint",
+ }),
+ }),
+ ),
),
Effect.provideService(
HttpClient.HttpClient,
@@ -70,15 +141,18 @@ describe("GitHubMediaFetch", () => {
]);
}).pipe(
Effect.provide(
- Layer.mock(GitHubCredentials.GitHubCredentials)({
- get: (host) =>
- Effect.succeed({
- host,
- token: Redacted.make("signed-in"),
- source: "gh" as const,
- fingerprint: "fingerprint",
- }),
- }),
+ Layer.merge(
+ Layer.mock(GitHubRest.GitHubRest)({}),
+ Layer.mock(GitHubCredentials.GitHubCredentials)({
+ get: (host) =>
+ Effect.succeed({
+ host,
+ token: Redacted.make("signed-in"),
+ source: "gh" as const,
+ fingerprint: "fingerprint",
+ }),
+ }),
+ ),
),
Effect.provideService(
HttpClient.HttpClient,
@@ -101,4 +175,68 @@ describe("GitHubMediaFetch", () => {
Effect.scoped,
);
});
+
+ it.effect(
+ "renders SSO uploads in the PR repository and reuses the signed URL for a range",
+ () => {
+ const url = "https://github.com/user-attachments/assets/explicit-pr-context";
+ const fixture = makeSsoFixture(url);
+ return Effect.gen(function* () {
+ const fetchMedia = yield* GitHubMediaFetch.make;
+ const request = { url, headers: {}, repository: PR_REPOSITORY };
+ expect((yield* fetchMedia(request))?.status).toBe(200);
+ expect((yield* fetchMedia({ ...request, headers: { range: "bytes=2-5" } }))?.status).toBe(
+ 206,
+ );
+ expect(fixture.rendered).toHaveLength(1);
+ expect(fixture.rendered[0]).toMatchObject({
+ host: "github.com",
+ method: "POST",
+ path: "markdown",
+ accept: "text/html",
+ body: { text: `![](${url})`, mode: "gfm", context: PR_REPOSITORY },
+ });
+ expect(fixture.requests).toEqual([
+ { url, authorization: "Bearer account-a" },
+ { url: fixture.signedUrls[0] },
+ { url: fixture.signedUrls[0], range: "bytes=2-5" },
+ ]);
+ }).pipe(Effect.provide(fixture.layer), Effect.scoped);
+ },
+ );
+
+ it.effect("renders a legacy upload in the repository its path names", () => {
+ const url = `https://github.com/${PR_REPOSITORY}/assets/123/legacy`;
+ const fixture = makeSsoFixture(url);
+ return Effect.gen(function* () {
+ const fetchMedia = yield* GitHubMediaFetch.make;
+ const response = yield* fetchMedia({ url, headers: {}, repository: "personal/unrelated" });
+ expect(response?.status).toBe(200);
+ expect(fixture.rendered.map((input) => input.body)).toEqual([
+ { text: `![](${url})`, mode: "gfm", context: PR_REPOSITORY },
+ ]);
+ }).pipe(Effect.provide(fixture.layer), Effect.scoped);
+ });
+
+ it.effect("renders a fresh signed URL after an account switch, early expiry, or its TTL", () => {
+ const url = "https://github.com/user-attachments/assets/cache-lifecycle";
+ const fixture = makeSsoFixture(url);
+ const request = { url, headers: {}, repository: PR_REPOSITORY };
+ return Effect.gen(function* () {
+ const fetchMedia = yield* GitHubMediaFetch.make;
+ yield* fetchMedia(request);
+ fixture.state.account = "account-b";
+ yield* fetchMedia(request);
+ expect(fixture.rendered).toHaveLength(2);
+
+ fixture.state.expired.add(fixture.signedUrls.at(-1)!);
+ expect((yield* fetchMedia(request))?.status).toBe(200);
+ expect(fixture.rendered).toHaveLength(3);
+
+ yield* TestClock.adjust("6 minutes");
+ yield* fetchMedia(request);
+ expect(fixture.rendered).toHaveLength(4);
+ expect(fixture.requests.at(-1)?.url).toBe(fixture.signedUrls.at(-1));
+ }).pipe(Effect.provide(fixture.layer), Effect.scoped);
+ });
});
diff --git a/packages/source-control-github/src/server/gitHubMediaFetch.ts b/packages/source-control-github/src/server/gitHubMediaFetch.ts
index 4dfb2a23e63..b03c0d327a6 100644
--- a/packages/source-control-github/src/server/gitHubMediaFetch.ts
+++ b/packages/source-control-github/src/server/gitHubMediaFetch.ts
@@ -1,4 +1,5 @@
import type { SourceControlMediaRequest } from "@t3tools/source-control-core/server/driver";
+import * as Clock from "effect/Clock";
import * as Effect from "effect/Effect";
import * as Redacted from "effect/Redacted";
import {
@@ -11,6 +12,7 @@ import {
import { GITHUB_CREDENTIALED_MEDIA_HOSTS } from "../shared/gitHubMedia.ts";
import * as GitHubCredentials from "./GitHubCredentials.ts";
+import * as GitHubRest from "./GitHubRest.ts";
/**
* Everything a redirect leads to — the presigned object stores GitHub hands assets off to —
@@ -18,51 +20,170 @@ import * as GitHubCredentials from "./GitHubCredentials.ts";
* bearer token.
*/
const isCredentialedHost = (url: string) => {
- try {
- return GITHUB_CREDENTIALED_MEDIA_HOSTS.has(new URL(url).hostname.toLowerCase());
- } catch {
- return false;
- }
+ const target = URL.parse(url);
+ return (
+ target !== null &&
+ target.protocol === "https:" &&
+ target.port === "" &&
+ GITHUB_CREDENTIALED_MEDIA_HOSTS.has(target.hostname.toLowerCase())
+ );
};
const MAX_REDIRECTS = 3;
const MANUAL_REDIRECT: RequestInit = { redirect: "manual" };
+const MEDIA_CONTENT_TYPE_PATTERN = /^(?:image|video|audio)\/[\w!#$&^.+-]+$/i;
+const ATTACHMENT_PATH_PATTERN =
+ /^\/(?:user-attachments\/assets\/[\w-]+|[^/]+\/[^/]+\/assets\/\d+\/[\w-]+)$/u;
+const LEGACY_ATTACHMENT_REPOSITORY_PATTERN = /^\/([^/]+\/[^/]+)\/assets\//u;
+const REPOSITORY_PATTERN = /^[\w.-]+\/[\w.-]+$/u;
+// GitHub's signed upload links last about five minutes.
+const SIGNED_URL_CACHE_TTL_MS = 4 * 60_000;
+const SIGNED_URL_CACHE_MAX_ENTRIES = 64;
+
+const contentTypeOf = (response: HttpClientResponse.HttpClientResponse) =>
+ response.headers["content-type"]?.split(";", 1)[0]?.trim().toLowerCase() ?? "";
+
+/** The repository an upload has to be rendered in, or `undefined` when it is not an upload. */
+const uploadRepository = (request: SourceControlMediaRequest) => {
+ const target = URL.parse(request.url);
+ if (
+ target === null ||
+ target.protocol !== "https:" ||
+ target.hostname !== "github.com" ||
+ target.port !== "" ||
+ target.username !== "" ||
+ target.password !== "" ||
+ !ATTACHMENT_PATH_PATTERN.test(target.pathname)
+ ) {
+ return undefined;
+ }
+ // A legacy upload names its repository; one the caller supplies cannot override that.
+ const repository =
+ LEGACY_ATTACHMENT_REPOSITORY_PATTERN.exec(target.pathname)?.[1] ?? request.repository;
+ // An unusable explicit context must not fall back to an unrelated checkout.
+ return repository !== undefined && REPOSITORY_PATTERN.test(repository) ? repository : undefined;
+};
export const make = Effect.gen(function* () {
const credentials = yield* GitHubCredentials.GitHubCredentials;
+ const rest = yield* GitHubRest.GitHubRest;
const httpClient = HttpClient.withScope(yield* HttpClient.HttpClient);
+ const signedUrls = new Map<string, { readonly at: number; readonly url: string }>();
// Both media hosts are served by github.com's account, which is the host `gh` stores it under.
// Read on every fetch, so media loads as soon as the user signs in.
- const token = Effect.suspend(() => credentials.get("github.com")).pipe(
- Effect.map((credential) => credential.token),
+ const credential = Effect.suspend(() => credentials.get("github.com")).pipe(
Effect.orElseSucceed(() => null),
);
- return Effect.fn("GitHubMediaFetch.fetchMedia")(function* (request: SourceControlMediaRequest) {
- const bearer = yield* token;
- let target = request.url;
+ const fetchFollowingRedirects = Effect.fnUntraced(function* (
+ url: string,
+ headers: Readonly<Record<string, string>>,
+ token: Redacted.Redacted<string> | null,
+ ) {
+ let target = url;
for (let hop = 0; ; hop += 1) {
const authorization =
- bearer !== null && isCredentialedHost(target) ? `Bearer ${Redacted.value(bearer)}` : null;
+ token !== null && isCredentialedHost(target) ? `Bearer ${Redacted.value(token)}` : null;
const response: HttpClientResponse.HttpClientResponse = yield* httpClient
.execute(
HttpClientRequest.get(target).pipe(
HttpClientRequest.setHeaders({
- ...request.headers,
+ ...headers,
// The bytes are streamed straight through, so never let an encoding layer in.
"accept-encoding": "identity",
...(authorization === null ? {} : { authorization }),
}),
),
)
- .pipe(Effect.provideService(FetchHttpClient.RequestInit, MANUAL_REDIRECT));
+ .pipe(
+ Effect.provideService(FetchHttpClient.RequestInit, MANUAL_REDIRECT),
+ // A signed URL's query string is a bearer capability; keep it out of tracing spans.
+ Effect.provideService(HttpClient.TracerDisabledWhen, () => true),
+ );
const location = response.headers.location;
if (response.status < 300 || response.status >= 400) return response;
if (!location || hop >= MAX_REDIRECTS) return null;
- const next = new URL(location, target);
- if (next.protocol !== "https:") return null;
+ const next = URL.parse(location, target);
+ if (next === null || next.protocol !== "https:" || next.username || next.password) {
+ return null;
+ }
target = next.toString();
}
});
+
+ // An organization with SAML SSO answers a token-authenticated upload request with its sign-in
+ // page. Rendering the link as markdown in the repository's context hands back a signed URL.
+ const renderSignedUrl = Effect.fnUntraced(function* (url: string, repository: string) {
+ const rendered = yield* rest
+ .rest({
+ host: "github.com",
+ operation: "renderMarkdown",
+ method: "POST",
+ path: "markdown",
+ accept: "text/html",
+ body: { text: `![](${url})`, mode: "gfm", context: repository },
+ })
+ .pipe(Effect.orElseSucceed(() => null));
+ if (rendered === null || rendered.status !== 200 || rendered.truncated) return null;
+ for (const match of rendered.body.matchAll(/<img\b[^>]*\ssrc\s*=\s*(["'])(.*?)\1/giu)) {
+ const signed = URL.parse(match[2]?.replaceAll("&amp;", "&") ?? "");
+ if (
+ signed?.protocol === "https:" &&
+ signed.hostname === "private-user-images.githubusercontent.com" &&
+ signed.port === "" &&
+ signed.username === "" &&
+ signed.password === "" &&
+ signed.searchParams.get("jwt")
+ ) {
+ return signed.toString();
+ }
+ }
+ return null;
+ });
+
+ return Effect.fn("GitHubMediaFetch.fetchMedia")(function* (request: SourceControlMediaRequest) {
+ const signedIn = yield* credential;
+ const repository = signedIn === null ? undefined : uploadRepository(request);
+ const cacheKey =
+ signedIn === null || repository === undefined
+ ? null
+ : [signedIn.fingerprint, repository, request.url].map(encodeURIComponent).join("|");
+ const now = yield* Clock.currentTimeMillis;
+ const cached = cacheKey === null ? undefined : signedUrls.get(cacheKey);
+ let response: HttpClientResponse.HttpClientResponse | null;
+ if (cacheKey !== null && cached !== undefined && now - cached.at < SIGNED_URL_CACHE_TTL_MS) {
+ response = yield* fetchFollowingRedirects(cached.url, request.headers, null);
+ if (response === null || response.status >= 400) signedUrls.delete(cacheKey);
+ // A signed URL can expire before its TTL; only then is a fresh one worth rendering.
+ if (response?.status !== 403) return response;
+ } else {
+ if (cacheKey !== null) signedUrls.delete(cacheKey);
+ response = yield* fetchFollowingRedirects(
+ request.url,
+ request.headers,
+ signedIn?.token ?? null,
+ );
+ if (response === null || contentTypeOf(response) !== "text/html") return response;
+ }
+ if (cacheKey === null || repository === undefined || signedIn === null) return response;
+ const signed = yield* renderSignedUrl(request.url, repository);
+ if (signed === null) return response;
+ const media = yield* fetchFollowingRedirects(signed, request.headers, null);
+ if (
+ (media?.status === 200 || media?.status === 206) &&
+ MEDIA_CONTENT_TYPE_PATTERN.test(contentTypeOf(media))
+ ) {
+ // The render can run under another account if the user switched mid-request.
+ const current = yield* credential;
+ if (current?.fingerprint === signedIn.fingerprint) {
+ if (signedUrls.size >= SIGNED_URL_CACHE_MAX_ENTRIES) {
+ const oldest = signedUrls.keys().next().value;
+ if (oldest !== undefined) signedUrls.delete(oldest);
+ }
+ signedUrls.set(cacheKey, { at: now, url: signed });
+ }
+ }
+ return media;
+ });
});