Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(alexandria): log feedback token rejections and the Exchange tool-feedback outcome (#5646)

Nicolas7e31458e66ff
diff --git a/apps/api/src/controllers/v2/feedback/alexandria.test.ts b/apps/api/src/controllers/v2/feedback/alexandria.test.ts
index d359de9cd..eb6df09e8 100644
--- a/apps/api/src/controllers/v2/feedback/alexandria.test.ts
+++ b/apps/api/src/controllers/v2/feedback/alexandria.test.ts
@@ -42,7 +42,9 @@ const parentRows = () => inserted("alexandria_feedback");
const providerRows = () => inserted("alexandria_feedback_providers");
const capabilityRows = () => inserted("alexandria_feedback_capabilities");
const callRows = () => inserted("alexandria_feedback_calls");
-vi.mock("../../../lib/logger", () => ({ logger: { error: mocks.logError } }));
+vi.mock("../../../lib/logger", () => ({
+ logger: { error: mocks.logError, warn: vi.fn() },
+}));
vi.mock("./record", () => ({
recordEndpointFeedback: mocks.recordEndpointFeedback,
}));
diff --git a/apps/api/src/controllers/v2/feedback/alexandria.ts b/apps/api/src/controllers/v2/feedback/alexandria.ts
index 4893ee5c3..75a145f12 100644
--- a/apps/api/src/controllers/v2/feedback/alexandria.ts
+++ b/apps/api/src/controllers/v2/feedback/alexandria.ts
@@ -7,7 +7,10 @@ import {
getRecentAlexandriaCalls,
hasRecentAlexandriaActivity,
} from "../../../lib/alexandria-activity";
-import { verifyFeedbackToken } from "../../../lib/alexandria-feedback-token";
+import {
+ checkFeedbackToken,
+ type FeedbackTokenRejection,
+} from "../../../lib/alexandria-feedback-token";
import { forwardExchangeToolFeedback } from "../../../lib/exchange";
import { logger } from "../../../lib/logger";
import { getScrapeZDR, getSearchZDR } from "../../../lib/zdr-helpers";
@@ -88,12 +91,16 @@ export async function recordAlexandriaFeedback(
];
const calls = new Map<string, FeedbackCall>();
let rejectedTokens = 0;
+ const rejectionReasons: Partial<Record<FeedbackTokenRejection, number>> = {};
for (const token of submittedTokens) {
- const claims = verifyFeedbackToken(token, teamId, now);
- if (!claims) {
+ const checked = checkFeedbackToken(token, teamId, now);
+ if (!checked.claims) {
rejectedTokens++;
+ rejectionReasons[checked.reason] =
+ (rejectionReasons[checked.reason] ?? 0) + 1;
continue;
}
+ const { claims } = checked;
const key = JSON.stringify([
claims.requestId,
claims.provider,
@@ -108,6 +115,15 @@ export async function recordAlexandriaFeedback(
});
}
const attribution = calls.size > 0 ? "token" : "window";
+ if (rejectedTokens > 0) {
+ logger.warn("Alexandria feedback tokens rejected", {
+ teamId,
+ submittedTokens: submittedTokens.length,
+ rejectedTokens,
+ reasons: rejectionReasons,
+ attribution,
+ });
+ }
if (attribution === "window") {
let withinWindow: boolean;
diff --git a/apps/api/src/lib/alexandria-feedback-token.test.ts b/apps/api/src/lib/alexandria-feedback-token.test.ts
index 66740115f..35c492b82 100644
--- a/apps/api/src/lib/alexandria-feedback-token.test.ts
+++ b/apps/api/src/lib/alexandria-feedback-token.test.ts
@@ -6,6 +6,7 @@ import { createHmac } from "node:crypto";
import { config } from "../config";
import {
MAX_FEEDBACK_TOKEN_LENGTH,
+ checkFeedbackToken,
issueFeedbackToken,
verifyFeedbackToken,
} from "./alexandria-feedback-token";
@@ -113,3 +114,32 @@ it("changes every token when the key rotates", () => {
config.EXCHANGE_INTERNAL_SECRET = "rotated";
expect(verifyFeedbackToken(token, teamId, now)).toBeNull();
});
+
+it("names the reason a token is rejected", () => {
+ const token = issueFeedbackToken(claims, now)!;
+ const [version, payload, signature] = token.split(".");
+ const flipped = Buffer.from(signature, "base64url");
+ flipped[0] ^= 1;
+ const reason = (value: string, at = now) =>
+ (checkFeedbackToken(value, teamId, at) as { reason?: string }).reason;
+ expect(checkFeedbackToken(token, teamId, now).claims).not.toBeNull();
+ expect(reason(`${token}.extra`)).toBe("malformed");
+ expect(reason(`${version}.${payload}.${flipped.toString("base64url")}`)).toBe(
+ "bad_signature",
+ );
+ expect(
+ (
+ checkFeedbackToken(
+ token,
+ "01933161-0000-7000-8000-000000000002",
+ now,
+ ) as {
+ reason?: string;
+ }
+ ).reason,
+ ).toBe("bad_signature");
+ expect(reason(token, now + 24 * 3600_000 + 1000)).toBe("expired");
+ expect(reason(token, now - 120_000)).toBe("not_yet_valid");
+ config.EXCHANGE_INTERNAL_SECRET = undefined as never;
+ expect(reason(token)).toBe("no_key");
+});
diff --git a/apps/api/src/lib/alexandria-feedback-token.ts b/apps/api/src/lib/alexandria-feedback-token.ts
index c0d99966b..a4981bbb9 100644
--- a/apps/api/src/lib/alexandria-feedback-token.ts
+++ b/apps/api/src/lib/alexandria-feedback-token.ts
@@ -55,28 +55,41 @@ export function issueFeedbackToken(
return token.length <= MAX_FEEDBACK_TOKEN_LENGTH ? token : undefined;
}
-/** The token's claims if it was issued to teamId, its signature holds and it is under 24 hours old; otherwise null. */
-export function verifyFeedbackToken(
+export type FeedbackTokenRejection =
+ | "no_key"
+ | "malformed"
+ | "bad_signature"
+ | "expired"
+ | "not_yet_valid";
+
+/** Verifies a token for teamId; a token issued to another team fails as bad_signature. */
+export function checkFeedbackToken(
token: string,
teamId: string,
now = Date.now(),
-): FeedbackTokenClaims | null {
+):
+ | { claims: FeedbackTokenClaims }
+ | { claims: null; reason: FeedbackTokenRejection } {
+ const reject = (reason: FeedbackTokenRejection) => ({
+ claims: null,
+ reason,
+ });
const key = signingKey();
- if (!key) return null;
+ if (!key) return reject("no_key");
const [version, payload, signature, ...rest] = token.split(".");
if (version !== VERSION || !payload || !signature || rest.length > 0)
- return null;
+ return reject("malformed");
const expected = sign(key, payload, teamId);
const given = Buffer.from(signature, "base64url");
if (given.length !== expected.length || !timingSafeEqual(given, expected))
- return null;
+ return reject("bad_signature");
let fields: unknown;
try {
fields = JSON.parse(Buffer.from(payload, "base64url").toString("utf8"));
} catch {
- return null;
+ return reject("malformed");
}
- if (!Array.isArray(fields) || fields.length !== 4) return null;
+ if (!Array.isArray(fields) || fields.length !== 4) return reject("malformed");
const [requestId, provider, capability, issuedAt] = fields;
if (
typeof requestId !== "string" ||
@@ -85,8 +98,18 @@ export function verifyFeedbackToken(
typeof issuedAt !== "number" ||
!Number.isInteger(issuedAt)
)
- return null;
+ return reject("malformed");
const age = now / 1000 - issuedAt;
- if (age > MAX_AGE_SEC || age < -CLOCK_SKEW_SEC) return null;
- return { requestId, provider, capability, issuedAt };
+ if (age > MAX_AGE_SEC) return reject("expired");
+ if (age < -CLOCK_SKEW_SEC) return reject("not_yet_valid");
+ return { claims: { requestId, provider, capability, issuedAt } };
+}
+
+/** The token's claims if it was issued to teamId, its signature holds and it is under 24 hours old; otherwise null. */
+export function verifyFeedbackToken(
+ token: string,
+ teamId: string,
+ now = Date.now(),
+): FeedbackTokenClaims | null {
+ return checkFeedbackToken(token, teamId, now).claims;
}
diff --git a/apps/api/src/lib/exchange.test.ts b/apps/api/src/lib/exchange.test.ts
index 148f354a5..e0effacbb 100644
--- a/apps/api/src/lib/exchange.test.ts
+++ b/apps/api/src/lib/exchange.test.ts
@@ -23,6 +23,7 @@ import {
ThirdPartyDataTermsRequiredError,
ThirdPartyDataUnsupportedOptionError,
} from "./exchange";
+import { logger } from "./logger";
vi.mock("undici", () => ({
fetch: vi.fn(),
@@ -1062,10 +1063,23 @@ describe("Exchange tool feedback forward", () => {
});
it("posts the rating and its calls with the internal secret and team header", async () => {
- vi.mocked(fetch).mockResolvedValue({ ok: true, status: 202 } as Awaited<
- ReturnType<typeof fetch>
- >);
+ const info = vi.spyOn(logger, "info");
+ vi.mocked(fetch).mockResolvedValue({
+ ok: true,
+ status: 202,
+ text: async () => JSON.stringify({ success: true, outcome: "counted" }),
+ } as unknown as Awaited<ReturnType<typeof fetch>>);
await forwardExchangeToolFeedback(feedback);
+ expect(info).toHaveBeenCalledWith(
+ "Exchange tool feedback recorded",
+ expect.objectContaining({
+ feedbackId: feedback.feedbackId,
+ outcome: "counted",
+ attribution: "token",
+ calls: 1,
+ }),
+ );
+ info.mockRestore();
expect(fetch).toHaveBeenCalledExactlyOnceWith(
"https://exchange.example/v1/tool-feedback",
expect.objectContaining({
diff --git a/apps/api/src/lib/exchange.ts b/apps/api/src/lib/exchange.ts
index 5b805f494..d30c3c785 100644
--- a/apps/api/src/lib/exchange.ts
+++ b/apps/api/src/lib/exchange.ts
@@ -921,7 +921,17 @@ export type ExchangeToolFeedback = {
}[];
};
-/** Forwards an Alexandria feedback rating to the Exchange. Never throws; failures are logged. */
+const toolFeedbackResponseSchema = z.object({
+ outcome: z.enum([
+ "duplicate",
+ "counted",
+ "window",
+ "unattributed",
+ "publisher",
+ ]),
+});
+
+/** Forwards an Alexandria feedback rating to the Exchange and logs its outcome. Never throws. */
export async function forwardExchangeToolFeedback(
input: ExchangeToolFeedback,
): Promise<void> {
@@ -946,7 +956,20 @@ export async function forwardExchangeToolFeedback(
...context,
statusCode: response.status,
});
+ return;
}
+ const text = await response.text();
+ let parsed: unknown = null;
+ try {
+ parsed = JSON.parse(text);
+ } catch {}
+ const outcome = toolFeedbackResponseSchema.safeParse(parsed);
+ rootLogger.info("Exchange tool feedback recorded", {
+ ...context,
+ attribution: input.attribution,
+ calls: input.calls.length,
+ outcome: outcome.success ? outcome.data.outcome : "unknown",
+ });
} catch (error) {
rootLogger.warn("Exchange tool feedback errored", { ...context, error });
}