Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(worker): mock DNS in url-normalization tests to remove CI flakiness (#16128)

Lysander Kieselc0bc50bb8089
diff --git a/worker/src/__tests__/llm-base-url-validation.test.ts b/worker/src/__tests__/llm-base-url-validation.test.ts
index f3a3d89066..b7e9ac3356 100644
--- a/worker/src/__tests__/llm-base-url-validation.test.ts
+++ b/worker/src/__tests__/llm-base-url-validation.test.ts
@@ -1,28 +1,5 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
-
-const { resolve4Mock, resolve6Mock, lookupMock } = vi.hoisted(() => ({
- resolve4Mock: vi.fn<(hostname: string) => Promise<string[]>>(),
- resolve6Mock: vi.fn<(hostname: string) => Promise<string[]>>(),
- lookupMock:
- vi.fn<
- (
- hostname: string,
- options: { all: true },
- ) => Promise<Array<{ address: string; family: 4 | 6 }>>
- >(),
-}));
-
-vi.mock("node:dns/promises", () => ({
- default: {
- resolve4: resolve4Mock,
- resolve6: resolve6Mock,
- lookup: lookupMock,
- },
- resolve4: resolve4Mock,
- resolve6: resolve6Mock,
- lookup: lookupMock,
-}));
-
+import dns from "node:dns/promises";
import {
validateLlmConnectionBaseURL,
type LlmBaseUrlValidationWhitelist,
@@ -37,14 +14,13 @@ const originalAllowedIpSegments =
describe("LLM base URL validation", () => {
beforeEach(() => {
- vi.clearAllMocks();
-
- resolve4Mock.mockRejectedValue(new Error("ENOTFOUND"));
- resolve6Mock.mockRejectedValue(new Error("ENODATA"));
- lookupMock.mockRejectedValue(new Error("ENOTFOUND"));
+ vi.spyOn(dns, "resolve4").mockRejectedValue(new Error("ENOTFOUND"));
+ vi.spyOn(dns, "resolve6").mockRejectedValue(new Error("ENODATA"));
+ vi.spyOn(dns, "lookup").mockRejectedValue(new Error("ENOTFOUND"));
});
afterEach(() => {
+ vi.restoreAllMocks();
(env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = originalCloudRegion;
(env as any).LANGFUSE_LLM_CONNECTION_WHITELISTED_HOST =
originalAllowedHosts;
@@ -83,15 +59,17 @@ describe("LLM base URL validation", () => {
it("should reject hostnames that resolve to blocked IPs through local lookup", async () => {
(env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;
- lookupMock.mockResolvedValue([{ address: "127.0.0.1", family: 4 }]);
+ vi.spyOn(dns, "lookup").mockResolvedValue([
+ { address: "127.0.0.1", family: 4 },
+ ]);
await expect(validateLlmConnectionBaseURL("http://vm/v1")).rejects.toThrow(
"Blocked IP address detected",
);
- expect(resolve4Mock).toHaveBeenCalledWith("vm");
- expect(resolve6Mock).toHaveBeenCalledWith("vm");
- expect(lookupMock).toHaveBeenCalledWith("vm", { all: true });
+ expect(dns.resolve4).toHaveBeenCalledWith("vm");
+ expect(dns.resolve6).toHaveBeenCalledWith("vm");
+ expect(dns.lookup).toHaveBeenCalledWith("vm", { all: true });
});
it("should allow explicitly allowlisted localhost hosts for self-hosted instances", async () => {
diff --git a/worker/src/__tests__/url-normalization.test.ts b/worker/src/__tests__/url-normalization.test.ts
index ccb392c7b8..6f72858cc2 100644
--- a/worker/src/__tests__/url-normalization.test.ts
+++ b/worker/src/__tests__/url-normalization.test.ts
@@ -1,6 +1,39 @@
-import { describe, it, expect } from "vitest";
+import { afterEach, beforeEach, describe, it, expect, vi } from "vitest";
+import dns from "node:dns/promises";
import { validateWebhookURL } from "@langfuse/shared/src/server";
+// Stub the dns module object, not vi.mock("node:dns/promises"): this file loads
+// the validator from @langfuse/shared's compiled output, whose require() vitest
+// does not intercept.
+const dnsError = (code: string, hostname: string) =>
+ Object.assign(new Error(`queryA ${code} ${hostname}`), { code });
+
+// Inside a cloud VM, resolver search domains turn a bare "metadata" label into
+// the metadata endpoint.
+const searchDomainHosts: Record<string, string> = {
+ metadata: "169.254.169.254",
+};
+
+beforeEach(() => {
+ vi.spyOn(dns, "resolve4").mockImplementation(async (hostname: string) => {
+ const ip = searchDomainHosts[hostname];
+ if (!ip) throw dnsError("ENOTFOUND", hostname);
+ return [ip];
+ });
+ vi.spyOn(dns, "resolve6").mockImplementation(async (hostname: string) => {
+ throw dnsError("ENODATA", hostname);
+ });
+ vi.spyOn(dns, "lookup").mockImplementation(async (hostname: string) => {
+ const ip = searchDomainHosts[hostname];
+ if (!ip) throw dnsError("ENOTFOUND", hostname);
+ return [{ address: ip, family: 4 }];
+ });
+});
+
+afterEach(() => {
+ vi.restoreAllMocks();
+});
+
describe("URL Normalization and Edge Cases", () => {
describe("URL encoding bypass attempts", () => {
it("should reject URL-encoded localhost", async () => {
@@ -214,7 +247,7 @@ describe("URL Normalization and Edge Cases", () => {
await expect(
validateWebhookURL(`https://${manySubdomains}/webhook`),
).rejects.toThrow(/DNS lookup failed/);
- }, 10000);
+ });
it("should handle empty hostname", async () => {
// This URL is parsed as hostname="webhook" by URL constructor
@@ -264,8 +297,10 @@ describe("URL Normalization and Edge Cases", () => {
];
for (const url of metadataAttempts) {
+ // "metadata" is blocked on the IP it resolves to, the rest on the
+ // hostname/literal IP. A DNS failure must not count as a pass.
await expect(validateWebhookURL(url)).rejects.toThrow(
- /Blocked|DNS lookup failed/,
+ /Blocked (hostname|IP address) detected/,
);
}
});
diff --git a/worker/src/__tests__/webhook-validation.test.ts b/worker/src/__tests__/webhook-validation.test.ts
index 0d99a20f53..32ddb0c833 100644
--- a/worker/src/__tests__/webhook-validation.test.ts
+++ b/worker/src/__tests__/webhook-validation.test.ts
@@ -1,28 +1,5 @@
-import { beforeEach, describe, it, expect, vi } from "vitest";
-
-const { resolve4Mock, resolve6Mock, lookupMock } = vi.hoisted(() => ({
- resolve4Mock: vi.fn<(hostname: string) => Promise<string[]>>(),
- resolve6Mock: vi.fn<(hostname: string) => Promise<string[]>>(),
- lookupMock:
- vi.fn<
- (
- hostname: string,
- options: { all: true },
- ) => Promise<Array<{ address: string; family: 4 | 6 }>>
- >(),
-}));
-
-vi.mock("node:dns/promises", () => ({
- default: {
- resolve4: resolve4Mock,
- resolve6: resolve6Mock,
- lookup: lookupMock,
- },
- resolve4: resolve4Mock,
- resolve6: resolve6Mock,
- lookup: lookupMock,
-}));
-
+import { afterEach, beforeEach, describe, it, expect, vi } from "vitest";
+import dns from "node:dns/promises";
import { validateWebhookURL } from "../../../packages/shared/src/server/webhooks/validation";
const nonexistentDomain = "this-domain-definitely-does-not-exist-12345.com";
@@ -31,17 +8,17 @@ const dnsError = (code: string, hostname: string) =>
Object.assign(new Error(`queryA ${code} ${hostname}`), { code });
beforeEach(() => {
- vi.clearAllMocks();
-
- resolve4Mock.mockImplementation(async (hostname: string) => {
+ vi.spyOn(dns, "resolve4").mockImplementation(async (hostname: string) => {
if (hostname === nonexistentDomain) {
throw dnsError("ENOTFOUND", hostname);
}
return ["93.184.216.34"];
});
- resolve6Mock.mockRejectedValue(dnsError("ENODATA", "mocked-hostname"));
- lookupMock.mockImplementation(async (hostname: string) => {
+ vi.spyOn(dns, "resolve6").mockImplementation(async (hostname: string) => {
+ throw dnsError("ENODATA", hostname);
+ });
+ vi.spyOn(dns, "lookup").mockImplementation(async (hostname: string) => {
if (hostname === nonexistentDomain) {
throw dnsError("ENOTFOUND", hostname);
}
@@ -50,6 +27,10 @@ beforeEach(() => {
});
});
+afterEach(() => {
+ vi.restoreAllMocks();
+});
+
describe("Webhook URL Validation", () => {
describe("validateWebhookURL", () => {
it("should accept valid public HTTPS URLs", async () => {
@@ -176,9 +157,9 @@ describe("Webhook URL Validation", () => {
await expect(
validateWebhookURL(`https://${nonexistentDomain}/hook`),
).rejects.toThrow("DNS lookup failed");
- expect(resolve4Mock).toHaveBeenCalledWith(nonexistentDomain);
- expect(resolve6Mock).toHaveBeenCalledWith(nonexistentDomain);
- expect(lookupMock).toHaveBeenCalledWith(nonexistentDomain, {
+ expect(dns.resolve4).toHaveBeenCalledWith(nonexistentDomain);
+ expect(dns.resolve6).toHaveBeenCalledWith(nonexistentDomain);
+ expect(dns.lookup).toHaveBeenCalledWith(nonexistentDomain, {
all: true,
});
});
@@ -218,9 +199,9 @@ describe("Webhook URL Validation", () => {
validateWebhookURL("http://тест.example.com/hook"),
).resolves.not.toThrow();
- expect(resolve4Mock).toHaveBeenCalledWith("xn--e1aybc.example.com");
- expect(resolve6Mock).toHaveBeenCalledWith("xn--e1aybc.example.com");
- expect(lookupMock).toHaveBeenCalledWith("xn--e1aybc.example.com", {
+ expect(dns.resolve4).toHaveBeenCalledWith("xn--e1aybc.example.com");
+ expect(dns.resolve6).toHaveBeenCalledWith("xn--e1aybc.example.com");
+ expect(dns.lookup).toHaveBeenCalledWith("xn--e1aybc.example.com", {
all: true,
});
});