Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(alexandria): log feedback token rejections and the Exchange tool-feedback outcome (#5646)
Nicolas7e31458e66ff
diff --git a/apps/api/src/controllers/v2/feedback/alexandria.test.ts b/apps/api/src/controllers/v2/feedback/alexandria.test.tsindex d359de9cd..eb6df09e8 100644--- a/apps/api/src/controllers/v2/feedback/alexandria.test.ts+++ b/apps/api/src/controllers/v2/feedback/alexandria.test.ts@@ -42,7 +42,9 @@ const parentRows = () => inserted("alexandria_feedback");const providerRows = () => inserted("alexandria_feedback_providers");const capabilityRows = () => inserted("alexandria_feedback_capabilities");const callRows = () => inserted("alexandria_feedback_calls");-vi.mock("../../../lib/logger", () => ({ logger: { error: mocks.logError } }));+vi.mock("../../../lib/logger", () => ({+ logger: { error: mocks.logError, warn: vi.fn() },+}));vi.mock("./record", () => ({recordEndpointFeedback: mocks.recordEndpointFeedback,}));diff --git a/apps/api/src/controllers/v2/feedback/alexandria.ts b/apps/api/src/controllers/v2/feedback/alexandria.tsindex 4893ee5c3..75a145f12 100644--- a/apps/api/src/controllers/v2/feedback/alexandria.ts+++ b/apps/api/src/controllers/v2/feedback/alexandria.ts@@ -7,7 +7,10 @@ import {getRecentAlexandriaCalls,hasRecentAlexandriaActivity,} from "../../../lib/alexandria-activity";-import { verifyFeedbackToken } from "../../../lib/alexandria-feedback-token";+import {+ checkFeedbackToken,+ type FeedbackTokenRejection,+} from "../../../lib/alexandria-feedback-token";import { forwardExchangeToolFeedback } from "../../../lib/exchange";import { logger } from "../../../lib/logger";import { getScrapeZDR, getSearchZDR } from "../../../lib/zdr-helpers";@@ -88,12 +91,16 @@ export async function recordAlexandriaFeedback(];const calls = new Map<string, FeedbackCall>();let rejectedTokens = 0;+ const rejectionReasons: Partial<Record<FeedbackTokenRejection, number>> = {};for (const token of submittedTokens) {- const claims = verifyFeedbackToken(token, teamId, now);- if (!claims) {+ const checked = checkFeedbackToken(token, teamId, now);+ if (!checked.claims) {rejectedTokens++;+ rejectionReasons[checked.reason] =+ (rejectionReasons[checked.reason] ?? 0) + 1;continue;}+ const { claims } = checked;const key = JSON.stringify([claims.requestId,claims.provider,@@ -108,6 +115,15 @@ export async function recordAlexandriaFeedback(});}const attribution = calls.size > 0 ? "token" : "window";+ if (rejectedTokens > 0) {+ logger.warn("Alexandria feedback tokens rejected", {+ teamId,+ submittedTokens: submittedTokens.length,+ rejectedTokens,+ reasons: rejectionReasons,+ attribution,+ });+ }if (attribution === "window") {let withinWindow: boolean;diff --git a/apps/api/src/lib/alexandria-feedback-token.test.ts b/apps/api/src/lib/alexandria-feedback-token.test.tsindex 66740115f..35c492b82 100644--- a/apps/api/src/lib/alexandria-feedback-token.test.ts+++ b/apps/api/src/lib/alexandria-feedback-token.test.ts@@ -6,6 +6,7 @@ import { createHmac } from "node:crypto";import { config } from "../config";import {MAX_FEEDBACK_TOKEN_LENGTH,+ checkFeedbackToken,issueFeedbackToken,verifyFeedbackToken,} from "./alexandria-feedback-token";@@ -113,3 +114,32 @@ it("changes every token when the key rotates", () => {config.EXCHANGE_INTERNAL_SECRET = "rotated";expect(verifyFeedbackToken(token, teamId, now)).toBeNull();});++it("names the reason a token is rejected", () => {+ const token = issueFeedbackToken(claims, now)!;+ const [version, payload, signature] = token.split(".");+ const flipped = Buffer.from(signature, "base64url");+ flipped[0] ^= 1;+ const reason = (value: string, at = now) =>+ (checkFeedbackToken(value, teamId, at) as { reason?: string }).reason;+ expect(checkFeedbackToken(token, teamId, now).claims).not.toBeNull();+ expect(reason(`${token}.extra`)).toBe("malformed");+ expect(reason(`${version}.${payload}.${flipped.toString("base64url")}`)).toBe(+ "bad_signature",+ );+ expect(+ (+ checkFeedbackToken(+ token,+ "01933161-0000-7000-8000-000000000002",+ now,+ ) as {+ reason?: string;+ }+ ).reason,+ ).toBe("bad_signature");+ expect(reason(token, now + 24 * 3600_000 + 1000)).toBe("expired");+ expect(reason(token, now - 120_000)).toBe("not_yet_valid");+ config.EXCHANGE_INTERNAL_SECRET = undefined as never;+ expect(reason(token)).toBe("no_key");+});diff --git a/apps/api/src/lib/alexandria-feedback-token.ts b/apps/api/src/lib/alexandria-feedback-token.tsindex c0d99966b..a4981bbb9 100644--- a/apps/api/src/lib/alexandria-feedback-token.ts+++ b/apps/api/src/lib/alexandria-feedback-token.ts@@ -55,28 +55,41 @@ export function issueFeedbackToken(return token.length <= MAX_FEEDBACK_TOKEN_LENGTH ? token : undefined;}-/** The token's claims if it was issued to teamId, its signature holds and it is under 24 hours old; otherwise null. */-export function verifyFeedbackToken(+export type FeedbackTokenRejection =+ | "no_key"+ | "malformed"+ | "bad_signature"+ | "expired"+ | "not_yet_valid";++/** Verifies a token for teamId; a token issued to another team fails as bad_signature. */+export function checkFeedbackToken(token: string,teamId: string,now = Date.now(),-): FeedbackTokenClaims | null {+):+ | { claims: FeedbackTokenClaims }+ | { claims: null; reason: FeedbackTokenRejection } {+ const reject = (reason: FeedbackTokenRejection) => ({+ claims: null,+ reason,+ });const key = signingKey();- if (!key) return null;+ if (!key) return reject("no_key");const [version, payload, signature, ...rest] = token.split(".");if (version !== VERSION || !payload || !signature || rest.length > 0)- return null;+ return reject("malformed");const expected = sign(key, payload, teamId);const given = Buffer.from(signature, "base64url");if (given.length !== expected.length || !timingSafeEqual(given, expected))- return null;+ return reject("bad_signature");let fields: unknown;try {fields = JSON.parse(Buffer.from(payload, "base64url").toString("utf8"));} catch {- return null;+ return reject("malformed");}- if (!Array.isArray(fields) || fields.length !== 4) return null;+ if (!Array.isArray(fields) || fields.length !== 4) return reject("malformed");const [requestId, provider, capability, issuedAt] = fields;if (typeof requestId !== "string" ||@@ -85,8 +98,18 @@ export function verifyFeedbackToken(typeof issuedAt !== "number" ||!Number.isInteger(issuedAt))- return null;+ return reject("malformed");const age = now / 1000 - issuedAt;- if (age > MAX_AGE_SEC || age < -CLOCK_SKEW_SEC) return null;- return { requestId, provider, capability, issuedAt };+ if (age > MAX_AGE_SEC) return reject("expired");+ if (age < -CLOCK_SKEW_SEC) return reject("not_yet_valid");+ return { claims: { requestId, provider, capability, issuedAt } };+}++/** The token's claims if it was issued to teamId, its signature holds and it is under 24 hours old; otherwise null. */+export function verifyFeedbackToken(+ token: string,+ teamId: string,+ now = Date.now(),+): FeedbackTokenClaims | null {+ return checkFeedbackToken(token, teamId, now).claims;}diff --git a/apps/api/src/lib/exchange.test.ts b/apps/api/src/lib/exchange.test.tsindex 148f354a5..e0effacbb 100644--- a/apps/api/src/lib/exchange.test.ts+++ b/apps/api/src/lib/exchange.test.ts@@ -23,6 +23,7 @@ import {ThirdPartyDataTermsRequiredError,ThirdPartyDataUnsupportedOptionError,} from "./exchange";+import { logger } from "./logger";vi.mock("undici", () => ({fetch: vi.fn(),@@ -1062,10 +1063,23 @@ describe("Exchange tool feedback forward", () => {});it("posts the rating and its calls with the internal secret and team header", async () => {- vi.mocked(fetch).mockResolvedValue({ ok: true, status: 202 } as Awaited<- ReturnType<typeof fetch>- >);+ const info = vi.spyOn(logger, "info");+ vi.mocked(fetch).mockResolvedValue({+ ok: true,+ status: 202,+ text: async () => JSON.stringify({ success: true, outcome: "counted" }),+ } as unknown as Awaited<ReturnType<typeof fetch>>);await forwardExchangeToolFeedback(feedback);+ expect(info).toHaveBeenCalledWith(+ "Exchange tool feedback recorded",+ expect.objectContaining({+ feedbackId: feedback.feedbackId,+ outcome: "counted",+ attribution: "token",+ calls: 1,+ }),+ );+ info.mockRestore();expect(fetch).toHaveBeenCalledExactlyOnceWith("https://exchange.example/v1/tool-feedback",expect.objectContaining({diff --git a/apps/api/src/lib/exchange.ts b/apps/api/src/lib/exchange.tsindex 5b805f494..d30c3c785 100644--- a/apps/api/src/lib/exchange.ts+++ b/apps/api/src/lib/exchange.ts@@ -921,7 +921,17 @@ export type ExchangeToolFeedback = {}[];};-/** Forwards an Alexandria feedback rating to the Exchange. Never throws; failures are logged. */+const toolFeedbackResponseSchema = z.object({+ outcome: z.enum([+ "duplicate",+ "counted",+ "window",+ "unattributed",+ "publisher",+ ]),+});++/** Forwards an Alexandria feedback rating to the Exchange and logs its outcome. Never throws. */export async function forwardExchangeToolFeedback(input: ExchangeToolFeedback,): Promise<void> {@@ -946,7 +956,20 @@ export async function forwardExchangeToolFeedback(...context,statusCode: response.status,});+ return;}+ const text = await response.text();+ let parsed: unknown = null;+ try {+ parsed = JSON.parse(text);+ } catch {}+ const outcome = toolFeedbackResponseSchema.safeParse(parsed);+ rootLogger.info("Exchange tool feedback recorded", {+ ...context,+ attribution: input.attribution,+ calls: input.calls.length,+ outcome: outcome.success ? outcome.data.outcome : "unknown",+ });} catch (error) {rootLogger.warn("Exchange tool feedback errored", { ...context, error });}