🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(worker): mock DNS in url-normalization tests to remove CI flakiness (#16128)
Lysander Kieselc0bc50bb8089
diff --git a/worker/src/__tests__/llm-base-url-validation.test.ts b/worker/src/__tests__/llm-base-url-validation.test.tsindex f3a3d89066..b7e9ac3356 100644--- a/worker/src/__tests__/llm-base-url-validation.test.ts+++ b/worker/src/__tests__/llm-base-url-validation.test.ts@@ -1,28 +1,5 @@import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";--const { resolve4Mock, resolve6Mock, lookupMock } = vi.hoisted(() => ({- resolve4Mock: vi.fn<(hostname: string) => Promise<string[]>>(),- resolve6Mock: vi.fn<(hostname: string) => Promise<string[]>>(),- lookupMock:- vi.fn<- (- hostname: string,- options: { all: true },- ) => Promise<Array<{ address: string; family: 4 | 6 }>>- >(),-}));--vi.mock("node:dns/promises", () => ({- default: {- resolve4: resolve4Mock,- resolve6: resolve6Mock,- lookup: lookupMock,- },- resolve4: resolve4Mock,- resolve6: resolve6Mock,- lookup: lookupMock,-}));-+import dns from "node:dns/promises";import {validateLlmConnectionBaseURL,type LlmBaseUrlValidationWhitelist,@@ -37,14 +14,13 @@ const originalAllowedIpSegments =describe("LLM base URL validation", () => {beforeEach(() => {- vi.clearAllMocks();-- resolve4Mock.mockRejectedValue(new Error("ENOTFOUND"));- resolve6Mock.mockRejectedValue(new Error("ENODATA"));- lookupMock.mockRejectedValue(new Error("ENOTFOUND"));+ vi.spyOn(dns, "resolve4").mockRejectedValue(new Error("ENOTFOUND"));+ vi.spyOn(dns, "resolve6").mockRejectedValue(new Error("ENODATA"));+ vi.spyOn(dns, "lookup").mockRejectedValue(new Error("ENOTFOUND"));});afterEach(() => {+ vi.restoreAllMocks();(env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = originalCloudRegion;(env as any).LANGFUSE_LLM_CONNECTION_WHITELISTED_HOST =originalAllowedHosts;@@ -83,15 +59,17 @@ describe("LLM base URL validation", () => {it("should reject hostnames that resolve to blocked IPs through local lookup", async () => {(env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;- lookupMock.mockResolvedValue([{ address: "127.0.0.1", family: 4 }]);+ vi.spyOn(dns, "lookup").mockResolvedValue([+ { address: "127.0.0.1", family: 4 },+ ]);await expect(validateLlmConnectionBaseURL("http://vm/v1")).rejects.toThrow("Blocked IP address detected",);- expect(resolve4Mock).toHaveBeenCalledWith("vm");- expect(resolve6Mock).toHaveBeenCalledWith("vm");- expect(lookupMock).toHaveBeenCalledWith("vm", { all: true });+ expect(dns.resolve4).toHaveBeenCalledWith("vm");+ expect(dns.resolve6).toHaveBeenCalledWith("vm");+ expect(dns.lookup).toHaveBeenCalledWith("vm", { all: true });});it("should allow explicitly allowlisted localhost hosts for self-hosted instances", async () => {diff --git a/worker/src/__tests__/url-normalization.test.ts b/worker/src/__tests__/url-normalization.test.tsindex ccb392c7b8..6f72858cc2 100644--- a/worker/src/__tests__/url-normalization.test.ts+++ b/worker/src/__tests__/url-normalization.test.ts@@ -1,6 +1,39 @@-import { describe, it, expect } from "vitest";+import { afterEach, beforeEach, describe, it, expect, vi } from "vitest";+import dns from "node:dns/promises";import { validateWebhookURL } from "@langfuse/shared/src/server";+// Stub the dns module object, not vi.mock("node:dns/promises"): this file loads+// the validator from @langfuse/shared's compiled output, whose require() vitest+// does not intercept.+const dnsError = (code: string, hostname: string) =>+ Object.assign(new Error(`queryA ${code} ${hostname}`), { code });++// Inside a cloud VM, resolver search domains turn a bare "metadata" label into+// the metadata endpoint.+const searchDomainHosts: Record<string, string> = {+ metadata: "169.254.169.254",+};++beforeEach(() => {+ vi.spyOn(dns, "resolve4").mockImplementation(async (hostname: string) => {+ const ip = searchDomainHosts[hostname];+ if (!ip) throw dnsError("ENOTFOUND", hostname);+ return [ip];+ });+ vi.spyOn(dns, "resolve6").mockImplementation(async (hostname: string) => {+ throw dnsError("ENODATA", hostname);+ });+ vi.spyOn(dns, "lookup").mockImplementation(async (hostname: string) => {+ const ip = searchDomainHosts[hostname];+ if (!ip) throw dnsError("ENOTFOUND", hostname);+ return [{ address: ip, family: 4 }];+ });+});++afterEach(() => {+ vi.restoreAllMocks();+});+describe("URL Normalization and Edge Cases", () => {describe("URL encoding bypass attempts", () => {it("should reject URL-encoded localhost", async () => {@@ -214,7 +247,7 @@ describe("URL Normalization and Edge Cases", () => {await expect(validateWebhookURL(`https://${manySubdomains}/webhook`),).rejects.toThrow(/DNS lookup failed/);- }, 10000);+ });it("should handle empty hostname", async () => {// This URL is parsed as hostname="webhook" by URL constructor@@ -264,8 +297,10 @@ describe("URL Normalization and Edge Cases", () => {];for (const url of metadataAttempts) {+ // "metadata" is blocked on the IP it resolves to, the rest on the+ // hostname/literal IP. A DNS failure must not count as a pass.await expect(validateWebhookURL(url)).rejects.toThrow(- /Blocked|DNS lookup failed/,+ /Blocked (hostname|IP address) detected/,);}});diff --git a/worker/src/__tests__/webhook-validation.test.ts b/worker/src/__tests__/webhook-validation.test.tsindex 0d99a20f53..32ddb0c833 100644--- a/worker/src/__tests__/webhook-validation.test.ts+++ b/worker/src/__tests__/webhook-validation.test.ts@@ -1,28 +1,5 @@-import { beforeEach, describe, it, expect, vi } from "vitest";--const { resolve4Mock, resolve6Mock, lookupMock } = vi.hoisted(() => ({- resolve4Mock: vi.fn<(hostname: string) => Promise<string[]>>(),- resolve6Mock: vi.fn<(hostname: string) => Promise<string[]>>(),- lookupMock:- vi.fn<- (- hostname: string,- options: { all: true },- ) => Promise<Array<{ address: string; family: 4 | 6 }>>- >(),-}));--vi.mock("node:dns/promises", () => ({- default: {- resolve4: resolve4Mock,- resolve6: resolve6Mock,- lookup: lookupMock,- },- resolve4: resolve4Mock,- resolve6: resolve6Mock,- lookup: lookupMock,-}));-+import { afterEach, beforeEach, describe, it, expect, vi } from "vitest";+import dns from "node:dns/promises";import { validateWebhookURL } from "../../../packages/shared/src/server/webhooks/validation";const nonexistentDomain = "this-domain-definitely-does-not-exist-12345.com";@@ -31,17 +8,17 @@ const dnsError = (code: string, hostname: string) =>Object.assign(new Error(`queryA ${code} ${hostname}`), { code });beforeEach(() => {- vi.clearAllMocks();-- resolve4Mock.mockImplementation(async (hostname: string) => {+ vi.spyOn(dns, "resolve4").mockImplementation(async (hostname: string) => {if (hostname === nonexistentDomain) {throw dnsError("ENOTFOUND", hostname);}return ["93.184.216.34"];});- resolve6Mock.mockRejectedValue(dnsError("ENODATA", "mocked-hostname"));- lookupMock.mockImplementation(async (hostname: string) => {+ vi.spyOn(dns, "resolve6").mockImplementation(async (hostname: string) => {+ throw dnsError("ENODATA", hostname);+ });+ vi.spyOn(dns, "lookup").mockImplementation(async (hostname: string) => {if (hostname === nonexistentDomain) {throw dnsError("ENOTFOUND", hostname);}@@ -50,6 +27,10 @@ beforeEach(() => {});});+afterEach(() => {+ vi.restoreAllMocks();+});+describe("Webhook URL Validation", () => {describe("validateWebhookURL", () => {it("should accept valid public HTTPS URLs", async () => {@@ -176,9 +157,9 @@ describe("Webhook URL Validation", () => {await expect(validateWebhookURL(`https://${nonexistentDomain}/hook`),).rejects.toThrow("DNS lookup failed");- expect(resolve4Mock).toHaveBeenCalledWith(nonexistentDomain);- expect(resolve6Mock).toHaveBeenCalledWith(nonexistentDomain);- expect(lookupMock).toHaveBeenCalledWith(nonexistentDomain, {+ expect(dns.resolve4).toHaveBeenCalledWith(nonexistentDomain);+ expect(dns.resolve6).toHaveBeenCalledWith(nonexistentDomain);+ expect(dns.lookup).toHaveBeenCalledWith(nonexistentDomain, {all: true,});});@@ -218,9 +199,9 @@ describe("Webhook URL Validation", () => {validateWebhookURL("http://тест.example.com/hook"),).resolves.not.toThrow();- expect(resolve4Mock).toHaveBeenCalledWith("xn--e1aybc.example.com");- expect(resolve6Mock).toHaveBeenCalledWith("xn--e1aybc.example.com");- expect(lookupMock).toHaveBeenCalledWith("xn--e1aybc.example.com", {+ expect(dns.resolve4).toHaveBeenCalledWith("xn--e1aybc.example.com");+ expect(dns.resolve6).toHaveBeenCalledWith("xn--e1aybc.example.com");+ expect(dns.lookup).toHaveBeenCalledWith("xn--e1aybc.example.com", {all: true,});});