Skip to content

armgit / alook

Public Imported from GitHub
0

Multi-Agent Collaboration for Team and Friends

fix(web): log link preview failures (#579)

Gustavo Ye17226f0e94df
diff --git a/src/web/src/app/api/community/link-preview/route.test.ts b/src/web/src/app/api/community/link-preview/route.test.ts
index b04140cb8..267b5d613 100644
--- a/src/web/src/app/api/community/link-preview/route.test.ts
+++ b/src/web/src/app/api/community/link-preview/route.test.ts
@@ -7,6 +7,7 @@ const mockKvGet = vi.fn()
const mockKvPut = vi.fn()
const mockR2Put = vi.fn()
const mockWaitUntil = vi.fn()
+const mockLogError = vi.fn()
vi.mock("@opennextjs/cloudflare", () => ({
getCloudflareContext: vi.fn(() => ({ ctx: { waitUntil: mockWaitUntil } })),
@@ -26,6 +27,10 @@ vi.mock("@/lib/rate-limit", () => ({
checkRateLimit: (...args: unknown[]) => mockCheckRateLimit(...args),
}))
+vi.mock("@/lib/logger", () => ({
+ log: { error: (...args: unknown[]) => mockLogError(...args) },
+}))
+
vi.mock("@/lib/middleware/auth", () => ({
withAuth: (handler: (...args: any[]) => unknown) => (req: NextRequest) => handler(req, {
env: {
@@ -44,6 +49,7 @@ vi.mock("@/lib/middleware/helpers", () => ({
}))
import { POST } from "./route"
+import { LinkPreviewFetchError } from "@/lib/community/link-preview-fetch"
function request(body: unknown): NextRequest {
return new NextRequest("http://localhost/api/community/link-preview", {
@@ -243,10 +249,16 @@ describe("POST /api/community/link-preview", () => {
{ expirationTtl: 21_600 },
)
expect(mockWaitUntil).toHaveBeenCalledWith(expect.any(Promise))
+ expect(mockLogError).not.toHaveBeenCalled()
})
it("negative-caches a bounded origin failure and degrades to no card", async () => {
- mockFetchLinkPreview.mockRejectedValue(new Error("timeout"))
+ mockFetchLinkPreview.mockRejectedValue(new LinkPreviewFetchError(
+ "origin unavailable",
+ "document_fetch",
+ "upstream_http_status",
+ 503,
+ ))
const response = await POST(request({ url: "https://example.com/" }))
@@ -256,6 +268,19 @@ describe("POST /api/community/link-preview", () => {
JSON.stringify({ preview: null, staleTimeSeconds: 300 }),
{ expirationTtl: 300 },
)
+ expect(mockLogError).toHaveBeenCalledOnce()
+ expect(mockLogError).toHaveBeenCalledWith("link_preview_failure", expect.objectContaining({
+ stage: "document_fetch",
+ errorCode: "upstream_http_status",
+ httpStatus: 503,
+ elapsedMs: expect.any(Number),
+ disposition: "negative_cache",
+ pageDigestPrefix: "0f115db062b7",
+ }))
+ const serialized = JSON.stringify(mockLogError.mock.calls[0]?.[1])
+ expect(serialized).not.toContain("https://example.com")
+ expect(serialized).not.toContain("origin unavailable")
+ expect(serialized).not.toContain("user_1")
})
it("swallows a background KV write failure", async () => {
@@ -295,6 +320,7 @@ describe("POST /api/community/link-preview", () => {
expect.objectContaining({ httpMetadata: { contentType: "application/json" } }),
)
expect(mockR2Put.mock.invocationCallOrder[0]).toBeLessThan(mockKvPut.mock.invocationCallOrder[0]!)
+ expect(mockLogError).not.toHaveBeenCalled()
})
it("omits a thumbnail and uses the recovery TTL when the manifest write fails", async () => {
@@ -317,6 +343,18 @@ describe("POST /api/community/link-preview", () => {
expect.stringContaining('"staleTimeSeconds":300'),
{ expirationTtl: 300 },
)
+ expect(mockLogError).toHaveBeenCalledOnce()
+ expect(mockLogError).toHaveBeenCalledWith("link_preview_failure", expect.objectContaining({
+ stage: "manifest_write",
+ errorCode: "manifest_write_failed",
+ elapsedMs: expect.any(Number),
+ disposition: "text_only_recovery_ttl",
+ pageDigestPrefix: "0f115db062b7",
+ }))
+ const serialized = JSON.stringify(mockLogError.mock.calls[0]?.[1])
+ expect(serialized).not.toContain("https://example.com")
+ expect(serialized).not.toContain("images.example.com")
+ expect(serialized).not.toContain("R2 unavailable")
})
it("enforces the authenticated preview rate limit", async () => {
diff --git a/src/web/src/app/api/community/link-preview/route.ts b/src/web/src/app/api/community/link-preview/route.ts
index bbc83d7e7..a61ed2124 100644
--- a/src/web/src/app/api/community/link-preview/route.ts
+++ b/src/web/src/app/api/community/link-preview/route.ts
@@ -3,7 +3,12 @@ import { getCloudflareContext } from "@opennextjs/cloudflare"
import { withAuth } from "@/lib/middleware/auth"
import { writeError, writeJSON } from "@/lib/middleware/helpers"
import { checkRateLimit } from "@/lib/rate-limit"
-import { fetchLinkPreview, normalizePublicPreviewUrl } from "@/lib/community/link-preview-fetch"
+import { log } from "@/lib/logger"
+import {
+ fetchLinkPreview,
+ LinkPreviewFetchError,
+ normalizePublicPreviewUrl,
+} from "@/lib/community/link-preview-fetch"
import {
linkPreviewPageDigest,
linkPreviewThumbnailUrl,
@@ -84,6 +89,21 @@ function cacheKey(pageDigest: string): string {
return `link-preview:v2:${pageDigest}`
}
+function elapsedSince(startedAt: number): number {
+ return Math.max(0, Date.now() - startedAt)
+}
+
+function logLinkPreviewFailure(fields: {
+ stage: string
+ errorCode: string
+ elapsedMs: number
+ disposition: "negative_cache" | "text_only_recovery_ttl"
+ pageDigestPrefix: string
+ httpStatus?: number
+}) {
+ log.error("link_preview_failure", fields)
+}
+
/**
* Independent, authenticated URL-unfurl boundary. It is deliberately not part
* of message list/send: text paints first, and a slow/failed origin only makes
@@ -129,11 +149,13 @@ export const POST = withAuth(async (req: NextRequest, ctx) => {
}
let entry: CacheEntry
+ const metadataStartedAt = Date.now()
try {
const fetched = await fetchLinkPreview(normalized)
const { thumbnailSource, ...preview } = fetched
let staleTimeSeconds: CacheEntry["staleTimeSeconds"] = POSITIVE_TTL_SECONDS
if (thumbnailSource) {
+ const manifestStartedAt = Date.now()
try {
await writeLinkPreviewThumbnailManifest({
bucket: ctx.env.COMMUNITY_MEDIA,
@@ -142,13 +164,29 @@ export const POST = withAuth(async (req: NextRequest, ctx) => {
})
preview.thumbnailUrl = linkPreviewThumbnailUrl(pageDigest)
} catch {
+ logLinkPreviewFailure({
+ stage: "manifest_write",
+ errorCode: "manifest_write_failed",
+ elapsedMs: elapsedSince(manifestStartedAt),
+ disposition: "text_only_recovery_ttl",
+ pageDigestPrefix: pageDigest.slice(0, 12),
+ })
// A capability is exposed only after its strongly-consistent manifest
// exists. Retry this degraded text-only preview after the negative TTL.
staleTimeSeconds = NEGATIVE_TTL_SECONDS
}
}
entry = { preview, staleTimeSeconds }
- } catch {
+ } catch (error) {
+ const failure = error instanceof LinkPreviewFetchError ? error : null
+ logLinkPreviewFailure({
+ stage: failure?.stage ?? "metadata_fetch",
+ errorCode: failure?.code ?? "unexpected_error",
+ elapsedMs: elapsedSince(metadataStartedAt),
+ disposition: "negative_cache",
+ pageDigestPrefix: pageDigest.slice(0, 12),
+ ...(failure?.httpStatus === undefined ? {} : { httpStatus: failure.httpStatus }),
+ })
entry = { preview: null, staleTimeSeconds: NEGATIVE_TTL_SECONDS }
}
diff --git a/src/web/src/lib/community/link-preview-fetch.test.ts b/src/web/src/lib/community/link-preview-fetch.test.ts
index dc301c7f2..80caa748e 100644
--- a/src/web/src/lib/community/link-preview-fetch.test.ts
+++ b/src/web/src/lib/community/link-preview-fetch.test.ts
@@ -1,7 +1,9 @@
import { afterEach, describe, expect, it, vi } from "vitest"
+import * as linkPreviewMobile from "link-preview-js/mobile"
import {
LINK_PREVIEW_LIMITS,
fetchLinkPreview,
+ LinkPreviewFetchError,
normalizePublicPreviewUrl,
} from "./link-preview-fetch"
@@ -167,7 +169,25 @@ describe("fetchLinkPreview", () => {
}), { headers: { "content-type": "application/json" } })))
await expect(fetchLinkPreview("https://youtu.be/dQw4w9WgXcQ"))
- .rejects.toThrow("YouTube oEmbed metadata missing")
+ .rejects.toMatchObject({
+ message: "YouTube oEmbed metadata missing",
+ stage: "metadata_parse",
+ code: "metadata_missing",
+ httpStatus: 200,
+ })
+ })
+
+ it("classifies malformed provider JSON as metadata parsing", async () => {
+ vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response("{", {
+ headers: { "content-type": "application/json" },
+ })))
+
+ await expect(fetchLinkPreview("https://youtu.be/dQw4w9WgXcQ"))
+ .rejects.toMatchObject({
+ stage: "metadata_parse",
+ code: "metadata_parse_failed",
+ httpStatus: 200,
+ })
})
it.each([
@@ -211,8 +231,15 @@ describe("fetchLinkPreview", () => {
}))
vi.stubGlobal("fetch", fetchMock)
- await expect(fetchLinkPreview("https://youtu.be/dQw4w9WgXcQ"))
- .rejects.toThrow("YouTube oEmbed response is not JSON")
+ const error = await fetchLinkPreview("https://youtu.be/dQw4w9WgXcQ").catch((caught) => caught)
+
+ expect(error).toBeInstanceOf(LinkPreviewFetchError)
+ expect(error).toMatchObject({
+ message: "YouTube oEmbed response is not JSON",
+ stage: "provider_metadata_fetch",
+ code: "unexpected_content_type",
+ httpStatus: 200,
+ })
expect(fetchMock).toHaveBeenCalledOnce()
expect(cancel).toHaveBeenCalledOnce()
})
@@ -243,7 +270,11 @@ describe("fetchLinkPreview", () => {
})))
const preview = fetchLinkPreview("https://youtu.be/dQw4w9WgXcQ")
- const rejected = expect(preview).rejects.toThrow("aborted")
+ const rejected = expect(preview).rejects.toMatchObject({
+ message: "aborted",
+ stage: "provider_metadata_fetch",
+ code: "timeout",
+ })
await vi.advanceTimersByTimeAsync(LINK_PREVIEW_LIMITS.timeoutMs)
await rejected
@@ -419,7 +450,12 @@ describe("fetchLinkPreview", () => {
vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response(body, { status: 503 })))
await expect(fetchLinkPreview("https://example.com/unavailable"))
- .rejects.toThrow("preview origin rejected request")
+ .rejects.toMatchObject({
+ message: "preview origin rejected request",
+ stage: "document_fetch",
+ code: "upstream_http_status",
+ httpStatus: 503,
+ })
expect(cancel).toHaveBeenCalledOnce()
})
@@ -507,6 +543,26 @@ describe("fetchLinkPreview", () => {
)))
await expect(fetchLinkPreview("https://example.com/untitled"))
- .rejects.toThrow("preview metadata missing")
+ .rejects.toMatchObject({
+ message: "preview metadata missing",
+ stage: "metadata_parse",
+ code: "metadata_missing",
+ })
+ })
+
+ it("classifies an HTML metadata parser failure", async () => {
+ vi.spyOn(linkPreviewMobile, "getPreviewFromContent")
+ .mockRejectedValueOnce(new Error("parser failed"))
+ vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response(
+ "<html><head><title>Example</title></head></html>",
+ { headers: { "content-type": "text/html" } },
+ )))
+
+ await expect(fetchLinkPreview("https://example.com/parser-failure"))
+ .rejects.toMatchObject({
+ message: "parser failed",
+ stage: "metadata_parse",
+ code: "metadata_parse_failed",
+ })
})
})
diff --git a/src/web/src/lib/community/link-preview-fetch.ts b/src/web/src/lib/community/link-preview-fetch.ts
index c5277de14..6d71fa511 100644
--- a/src/web/src/lib/community/link-preview-fetch.ts
+++ b/src/web/src/lib/community/link-preview-fetch.ts
@@ -4,6 +4,21 @@ import type { LinkPreview } from "./link-preview"
export type FetchedLinkPreview = LinkPreview & { thumbnailSource?: string }
+export type LinkPreviewFetchStage = "provider_metadata_fetch" | "document_fetch" | "metadata_parse"
+
+export class LinkPreviewFetchError extends Error {
+ readonly name = "LinkPreviewFetchError"
+
+ constructor(
+ message: string,
+ readonly stage: LinkPreviewFetchStage,
+ readonly code: string,
+ readonly httpStatus?: number,
+ ) {
+ super(message)
+ }
+}
+
const MAX_URL_LENGTH = 2_048
const MAX_HTML_BYTES = 128 * 1024
const MAX_OEMBED_BYTES = 16 * 1024
@@ -199,17 +214,45 @@ async function fetchYouTubeOEmbed(original: URL, videoId: string, signal: AbortS
})
if (!response.ok) {
await response.body?.cancel().catch(() => {})
- throw new Error("YouTube oEmbed rejected request")
+ throw new LinkPreviewFetchError(
+ "YouTube oEmbed rejected request",
+ "provider_metadata_fetch",
+ "upstream_http_status",
+ response.status,
+ )
}
const contentType = response.headers.get("content-type")?.toLowerCase() ?? ""
if (!/^application\/(?:[\w.+-]+\+)?json(?:;|$)/.test(contentType)) {
await response.body?.cancel().catch(() => {})
- throw new Error("YouTube oEmbed response is not JSON")
+ throw new LinkPreviewFetchError(
+ "YouTube oEmbed response is not JSON",
+ "provider_metadata_fetch",
+ "unexpected_content_type",
+ response.status,
+ )
}
- const record = JSON.parse(await readBoundedText(response, MAX_OEMBED_BYTES)) as Record<string, unknown>
+ let record: Record<string, unknown>
+ try {
+ record = JSON.parse(await readBoundedText(response, MAX_OEMBED_BYTES)) as Record<string, unknown>
+ } catch (error) {
+ const message = error instanceof Error ? error.message : "provider metadata parse failed"
+ throw new LinkPreviewFetchError(
+ message,
+ "metadata_parse",
+ signal.aborted ? "timeout" : "metadata_parse_failed",
+ response.status,
+ )
+ }
const title = cleanText(record.title, 160)
- if (!title) throw new Error("YouTube oEmbed metadata missing")
+ if (!title) {
+ throw new LinkPreviewFetchError(
+ "YouTube oEmbed metadata missing",
+ "metadata_parse",
+ "metadata_missing",
+ response.status,
+ )
+ }
const siteName = cleanText(record.provider_name, 80) ?? "YouTube"
const authorName = cleanText(record.author_name, 80)
const thumbnailSource = cleanThumbnailSource(record.thumbnail_url, original)
@@ -274,20 +317,44 @@ async function fetchHtml(start: URL, signal: AbortSignal): Promise<{ html: strin
if (isRedirect(response.status)) {
await response.body?.cancel().catch(() => {})
- if (redirects === MAX_REDIRECTS) throw new Error("too many preview redirects")
+ if (redirects === MAX_REDIRECTS) {
+ throw new LinkPreviewFetchError(
+ "too many preview redirects",
+ "document_fetch",
+ "redirect_limit",
+ response.status,
+ )
+ }
const location = response.headers.get("location")
- if (!location) throw new Error("invalid preview redirect")
+ if (!location) {
+ throw new LinkPreviewFetchError(
+ "invalid preview redirect",
+ "document_fetch",
+ "invalid_redirect",
+ response.status,
+ )
+ }
current = normalizePublicPreviewUrl(new URL(location, current).href)
continue
}
if (!response.ok) {
await response.body?.cancel().catch(() => {})
- throw new Error("preview origin rejected request")
+ throw new LinkPreviewFetchError(
+ "preview origin rejected request",
+ "document_fetch",
+ "upstream_http_status",
+ response.status,
+ )
}
const contentType = response.headers.get("content-type")?.toLowerCase() ?? ""
if (!/^text\/html(?:;|$)|^application\/xhtml\+xml(?:;|$)/.test(contentType)) {
await response.body?.cancel().catch(() => {})
- throw new Error("preview response is not HTML")
+ throw new LinkPreviewFetchError(
+ "preview response is not HTML",
+ "document_fetch",
+ "unexpected_content_type",
+ response.status,
+ )
}
return { html: await readBoundedHtml(response), finalUrl: current, contentType }
}
@@ -302,12 +369,16 @@ export async function fetchLinkPreview(input: string): Promise<FetchedLinkPrevie
const original = normalizePublicPreviewUrl(input)
const controller = new AbortController()
const timer = setTimeout(() => controller.abort(), TOTAL_TIMEOUT_MS)
+ let stage: LinkPreviewFetchStage = "document_fetch"
try {
const youtubeId = youtubeVideoId(original)
if (youtubeId) {
+ stage = "provider_metadata_fetch"
return await fetchYouTubeOEmbed(original, youtubeId, controller.signal)
}
+ stage = "document_fetch"
const { html, finalUrl, contentType } = await fetchHtml(original, controller.signal)
+ stage = "metadata_parse"
const parsed = await getPreviewFromContent({
url: finalUrl.href,
data: html,
@@ -336,7 +407,9 @@ export async function fetchLinkPreview(input: string): Promise<FetchedLinkPrevie
const thumbnailSource = Array.isArray(record.images)
? cleanThumbnailSource(record.images[0], finalUrl)
: undefined
- if (!parsedTitle && !description) throw new Error("preview metadata missing")
+ if (!parsedTitle && !description) {
+ throw new LinkPreviewFetchError("preview metadata missing", "metadata_parse", "metadata_missing")
+ }
const title = parsedTitle ?? siteName ?? original.hostname
return {
url: original.href,
@@ -346,6 +419,15 @@ export async function fetchLinkPreview(input: string): Promise<FetchedLinkPrevie
...(siteName ? { siteName } : {}),
...(thumbnailSource ? { thumbnailSource } : {}),
}
+ } catch (error) {
+ if (error instanceof LinkPreviewFetchError) throw error
+ const message = error instanceof Error ? error.message : "link preview fetch failed"
+ const code = controller.signal.aborted
+ ? "timeout"
+ : stage === "metadata_parse"
+ ? "metadata_parse_failed"
+ : "network_or_body_failed"
+ throw new LinkPreviewFetchError(message, stage, code)
} finally {
clearTimeout(timer)
}