Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): add GET /exchange/recommendations from team usage signals

rakshith484aea8cc3e9ed
diff --git a/apps/api/src/__tests__/routes/exchange-recommendations.routes.test.ts b/apps/api/src/__tests__/routes/exchange-recommendations.routes.test.ts
new file mode 100644
index 000000000..319e41045
--- /dev/null
+++ b/apps/api/src/__tests__/routes/exchange-recommendations.routes.test.ts
@@ -0,0 +1,393 @@
+import express from "express";
+import request from "supertest";
+
+// The real exchangeRouter, authMiddleware and auth.ts; only I/O is mocked.
+const mocks = vi.hoisted(() => ({
+ authChunk: vi.fn(),
+ upstreamFetch: vi.fn(),
+ clickhouse: null as { query: ReturnType<typeof vi.fn> } | null,
+ query: vi.fn(),
+}));
+
+vi.mock("../../lib/clickhouse-client", () => ({
+ get clickhouseClient() {
+ return mocks.clickhouse;
+ },
+ chInsert: vi.fn(),
+}));
+vi.mock("../../lib/billing-country", () => ({
+ billingCountryForTeam: vi.fn(async () => undefined),
+}));
+vi.mock("../../db/connection", () => ({ db: {}, dbRr: {}, dbIndex: {} }));
+vi.mock("../../db/rpc", () => ({
+ authCreditUsageChunk: mocks.authChunk,
+ authCreditUsageChunkFromTeam: vi.fn(),
+ getAgentFreeRequestsLeft: vi.fn(),
+}));
+vi.mock("../../services/redis", () => ({
+ getValue: vi.fn().mockResolvedValue(null),
+ setValue: vi.fn(),
+ deleteKey: vi.fn(),
+}));
+vi.mock("../../services/redlock", () => ({
+ redlock: { using: vi.fn(async (_k, _t, _o, fn) => fn({ aborted: false })) },
+}));
+vi.mock("ioredis", () => ({ default: class {} }));
+vi.mock("../../services/rate-limiter", async importOriginal => ({
+ ...(await importOriginal<typeof import("../../services/rate-limiter")>()),
+ getRateLimiter: vi.fn(() => ({ consume: vi.fn() })),
+ getAutumnRateLimiter: vi.fn(() => ({ consume: vi.fn() })),
+ redisRateLimitClient: {
+ set: vi.fn().mockResolvedValue("OK"),
+ get: vi.fn().mockResolvedValue(null),
+ del: vi.fn().mockResolvedValue(1),
+ },
+}));
+vi.mock("../../services/autumn/autumn.service", async importOriginal => {
+ const actual =
+ await importOriginal<
+ typeof import("../../services/autumn/autumn.service")
+ >();
+ return {
+ ...actual,
+ autumnService: {
+ getTeamLimits: vi.fn().mockResolvedValue({
+ concurrency_limit: 2,
+ rate_limit_multiplier: 1,
+ }),
+ },
+ };
+});
+vi.mock("../../services/agent-sponsor", () => ({
+ getAgentSponsorStatus: vi.fn().mockResolvedValue(null),
+}));
+vi.mock("../../services/queue-service", () => ({
+ getRedisConnection: vi.fn(() => ({ sadd: vi.fn() })),
+ getBillingQueue: vi.fn(() => ({ add: vi.fn() })),
+}));
+vi.mock("../../services/logging/log_job", () => ({
+ logRequest: vi.fn(),
+ logProviderScrape: vi.fn(),
+}));
+vi.mock("undici", async importOriginal => ({
+ ...(await importOriginal<typeof import("undici")>()),
+ fetch: mocks.upstreamFetch,
+}));
+
+import { config } from "../../config";
+import { logger } from "../../lib/logger";
+import { exchangeRouter } from "../../routes/exchange";
+
+const KEY = "44444444-4444-4444-8444-444444444444";
+const TEAM_ID = "11111111-1111-4111-8111-111111111111";
+const OTHER_TEAM_ID = "22222222-2222-4222-8222-222222222222";
+
+const RECOMMENDATIONS = {
+ success: true,
+ data: {
+ windowDays: 30,
+ items: [
+ {
+ provider: "youtube",
+ name: "YouTube",
+ description: "Videos",
+ categories: ["media"],
+ toolCount: 3,
+ kind: "website",
+ basis: "Matches sites you scrape (youtube.com)",
+ },
+ ],
+ },
+};
+
+const app = express();
+app.use(express.json());
+app.use("/exchange", exchangeRouter);
+
+function rowsAre(rows: { kind: string; target: string; n: number }[]) {
+ mocks.query.mockResolvedValue({ json: vi.fn().mockResolvedValue(rows) });
+}
+
+function clickhouseQuery() {
+ return mocks.query.mock.calls[0][0] as {
+ query: string;
+ query_params: Record<string, unknown>;
+ clickhouse_settings: Record<string, unknown>;
+ };
+}
+
+function upstream(): { url: string; init: RequestInit } {
+ const [url, init] = mocks.upstreamFetch.mock.calls[0]!;
+ return { url: String(url), init: init as RequestInit };
+}
+
+function forwardedBody() {
+ return JSON.parse(String(upstream().init.body));
+}
+
+describe("GET /exchange/recommendations", () => {
+ const original = {
+ USE_DB_AUTHENTICATION: config.USE_DB_AUTHENTICATION,
+ FIRE_EXCHANGE_URL: config.FIRE_EXCHANGE_URL,
+ };
+
+ beforeEach(() => {
+ vi.clearAllMocks();
+ config.USE_DB_AUTHENTICATION = true;
+ config.FIRE_EXCHANGE_URL = "https://exchange.test/";
+ mocks.clickhouse = { query: mocks.query };
+ mocks.authChunk.mockImplementation(
+ async (_db: unknown, key: string, purpose = "general") =>
+ key === KEY && purpose === "general"
+ ? [
+ {
+ api_key: KEY,
+ api_key_id: 9,
+ api_key_id_text: "9",
+ team_id: TEAM_ID,
+ org_id: "org-recommendations",
+ is_banned: false,
+ flags: {},
+ },
+ ]
+ : [],
+ );
+ mocks.upstreamFetch.mockImplementation(
+ async () =>
+ new Response(JSON.stringify(RECOMMENDATIONS), {
+ status: 200,
+ headers: { "content-type": "application/json" },
+ }),
+ );
+ rowsAre([]);
+ });
+
+ afterEach(() => {
+ Object.assign(config, original);
+ });
+
+ it("forwards the authenticated team's hosts and kinds and returns the Exchange's answer", async () => {
+ rowsAre([
+ { kind: "scrape", target: "youtube.com", n: 12 },
+ { kind: "crawl", target: "YouTube.com.", n: 3 },
+ { kind: "map", target: "docs.example.org", n: 5 },
+ { kind: "scrape", target: "not a host", n: 4 },
+ { kind: "scrape", target: "", n: 2 },
+ { kind: "search", target: "", n: 7 },
+ { kind: "research_paper_search", target: "", n: 4 },
+ ]);
+
+ const response = await request(app)
+ .get("/exchange/recommendations")
+ .set("authorization", `Bearer ${KEY}`)
+ .set("x-request-id", "req-1");
+
+ expect(response.status).toBe(200);
+ expect(response.body).toEqual(RECOMMENDATIONS);
+
+ const query = clickhouseQuery();
+ expect(query.query).toContain("FROM requests");
+ expect(query.query).toContain("team_id = {teamId: UUID}");
+ expect(query.query).toContain("kind != 'alexandria'");
+ expect(query.query).toContain("LIMIT {rowsPerKind: UInt32} BY kind");
+ expect(query.query_params).toMatchObject({
+ teamId: TEAM_ID,
+ urlKinds: ["scrape", "batch_scrape", "crawl", "map", "extract"],
+ redacted: "<redacted due to zero data retention>",
+ rowsPerKind: 200,
+ });
+ expect(query.clickhouse_settings).toEqual({
+ max_execution_time: 2,
+ timeout_overflow_mode: "break",
+ });
+
+ const { url, init } = upstream();
+ expect(url).toBe("https://exchange.test/v1/recommendations");
+ expect(init.method).toBe("POST");
+ expect(init.headers).toEqual({
+ "content-type": "application/json",
+ "x-exchange-team-id": TEAM_ID,
+ "x-request-id": "req-1",
+ });
+ expect(forwardedBody()).toEqual({
+ hosts: [
+ { host: "youtube.com", requests: 15 },
+ { host: "docs.example.org", requests: 5 },
+ ],
+ kinds: {
+ scrape: 18,
+ crawl: 3,
+ map: 5,
+ search: 7,
+ research_paper_search: 4,
+ },
+ });
+ });
+
+ it("ignores a team named by the caller in headers, query or body", async () => {
+ await request(app)
+ .get(
+ `/exchange/recommendations?team_id=${OTHER_TEAM_ID}&teamId=${OTHER_TEAM_ID}`,
+ )
+ .set("authorization", `Bearer ${KEY}`)
+ .set("x-exchange-team-id", OTHER_TEAM_ID)
+ .send({ teamId: OTHER_TEAM_ID, team_id: OTHER_TEAM_ID });
+
+ expect(clickhouseQuery().query_params.teamId).toBe(TEAM_ID);
+ expect(upstream().init.headers).toMatchObject({
+ "x-exchange-team-id": TEAM_ID,
+ });
+ expect(JSON.stringify(forwardedBody())).not.toContain(OTHER_TEAM_ID);
+ });
+
+ it("refuses an unauthenticated caller before reading anything", async () => {
+ const response = await request(app).get("/exchange/recommendations");
+ expect(response.status).toBe(401);
+ expect(mocks.query).not.toHaveBeenCalled();
+ expect(mocks.upstreamFetch).not.toHaveBeenCalled();
+ });
+
+ it.each(["", "?searches=false", "?searches=1"])(
+ "reads no search text without searches=true (%s)",
+ async suffix => {
+ rowsAre([
+ { kind: "search", target: "", n: 3 },
+ { kind: "scrape", target: "example.com", n: 1 },
+ ]);
+
+ await request(app)
+ .get(`/exchange/recommendations${suffix}`)
+ .set("authorization", `Bearer ${KEY}`);
+
+ expect(clickhouseQuery().query).not.toContain("kind = 'search'");
+ expect(clickhouseQuery().query).not.toContain("trimBoth");
+ const body = forwardedBody();
+ expect(body).not.toHaveProperty("searches");
+ expect(body.kinds).toEqual({ search: 3, scrape: 1 });
+ },
+ );
+
+ it("includes the team's search queries only with searches=true", async () => {
+ rowsAre([
+ { kind: "search", target: "fed interest rates", n: 4 },
+ { kind: "search", target: "site:youtube.com cooking", n: 2 },
+ ...Array.from({ length: 60 }, (_, i) => ({
+ kind: "search",
+ target: `query ${i}`,
+ n: 1,
+ })),
+ ]);
+
+ await request(app)
+ .get("/exchange/recommendations?searches=true")
+ .set("authorization", `Bearer ${KEY}`);
+
+ expect(clickhouseQuery().query).toContain(
+ "kind = 'search', trimBoth(target_hint)",
+ );
+ const body = forwardedBody();
+ expect(body.searches).toHaveLength(50);
+ expect(body.searches.slice(0, 2)).toEqual([
+ { query: "fed interest rates", requests: 4 },
+ { query: "site:youtube.com cooking", requests: 2 },
+ ]);
+ expect(body.kinds).toEqual({ search: 66 });
+ });
+
+ it("forwards at most 100 hosts, busiest first", async () => {
+ rowsAre(
+ Array.from({ length: 150 }, (_, i) => ({
+ kind: "scrape",
+ target: `site${i}.example.com`,
+ n: i + 1,
+ })),
+ );
+
+ await request(app)
+ .get("/exchange/recommendations")
+ .set("authorization", `Bearer ${KEY}`);
+
+ const { hosts } = forwardedBody();
+ expect(hosts).toHaveLength(100);
+ expect(hosts[0]).toEqual({ host: "site149.example.com", requests: 150 });
+ expect(hosts[99]).toEqual({ host: "site50.example.com", requests: 51 });
+ });
+
+ it("forwards empty signals when ClickHouse is not configured", async () => {
+ mocks.clickhouse = null;
+
+ const response = await request(app)
+ .get("/exchange/recommendations?searches=true")
+ .set("authorization", `Bearer ${KEY}`);
+
+ expect(response.status).toBe(200);
+ expect(forwardedBody()).toEqual({ hosts: [], searches: [], kinds: {} });
+ });
+
+ it("forwards empty signals and warns without query text when ClickHouse fails", async () => {
+ const warn = vi.spyOn(logger, "warn");
+ mocks.query.mockRejectedValue(new Error("Timeout exceeded"));
+
+ const response = await request(app)
+ .get("/exchange/recommendations")
+ .set("authorization", `Bearer ${KEY}`);
+
+ expect(response.status).toBe(200);
+ expect(response.body).toEqual(RECOMMENDATIONS);
+ expect(forwardedBody()).toEqual({ hosts: [], kinds: {} });
+ expect(warn).toHaveBeenCalledWith("Recommendation signals unavailable", {
+ module: "alexandria/recommendation-signals",
+ teamId: TEAM_ID,
+ error: "Timeout exceeded",
+ });
+ warn.mockRestore();
+ });
+
+ it("passes an Exchange 404 through verbatim", async () => {
+ mocks.upstreamFetch.mockResolvedValue(
+ new Response(JSON.stringify({ success: false, error: "Not found" }), {
+ status: 404,
+ headers: { "content-type": "application/json" },
+ }),
+ );
+
+ const response = await request(app)
+ .get("/exchange/recommendations")
+ .set("authorization", `Bearer ${KEY}`);
+
+ expect(response.status).toBe(404);
+ expect(response.body).toEqual({ success: false, error: "Not found" });
+ });
+
+ it("answers 503 without reading or forwarding when the Exchange is not configured", async () => {
+ config.FIRE_EXCHANGE_URL = undefined;
+
+ const response = await request(app)
+ .get("/exchange/recommendations")
+ .set("authorization", `Bearer ${KEY}`);
+
+ expect(response.status).toBe(503);
+ expect(response.body).toEqual({
+ success: false,
+ error: "This endpoint is not available.",
+ });
+ expect(mocks.query).not.toHaveBeenCalled();
+ expect(mocks.upstreamFetch).not.toHaveBeenCalled();
+ });
+
+ it("answers 504 when the Exchange times out and 502 when it cannot be reached", async () => {
+ mocks.upstreamFetch.mockRejectedValueOnce(
+ new DOMException("timed out", "TimeoutError"),
+ );
+ const timedOut = await request(app)
+ .get("/exchange/recommendations")
+ .set("authorization", `Bearer ${KEY}`);
+ expect(timedOut.status).toBe(504);
+
+ mocks.upstreamFetch.mockRejectedValueOnce(new Error("ECONNREFUSED"));
+ const unreachable = await request(app)
+ .get("/exchange/recommendations")
+ .set("authorization", `Bearer ${KEY}`);
+ expect(unreachable.status).toBe(502);
+ });
+});
diff --git a/apps/api/src/lib/world-id.ts b/apps/api/src/lib/world-id.ts
index bca5807ed..292735236 100644
--- a/apps/api/src/lib/world-id.ts
+++ b/apps/api/src/lib/world-id.ts
@@ -68,11 +68,11 @@ function listsSubject(list: string | undefined, subjectHash: string): boolean {
.includes(subjectHash);
}
-export function isWorldIdSubjectDenied(subjectHash: string): boolean {
+function isWorldIdSubjectDenied(subjectHash: string): boolean {
return listsSubject(config.WORLD_ID_DENIED_SUBJECTS, subjectHash);
}
-export function isWorldIdSubjectAllowed(subjectHash: string): boolean {
+function isWorldIdSubjectAllowed(subjectHash: string): boolean {
if (isWorldIdSubjectDenied(subjectHash)) return false;
const allowed = config.WORLD_ID_ALLOWED_SUBJECTS;
return !allowed || listsSubject(allowed, subjectHash);
@@ -304,7 +304,7 @@ const TOKEN_REJECTIONS = [
* Validates a World ID ID token and returns its subject hash, or null when the
* token is rejected. Throws when the issuer's keys can't be fetched.
*/
-export async function verifyWorldIdIdToken(
+async function verifyWorldIdIdToken(
idToken: string,
): Promise<string | null> {
try {
diff --git a/apps/api/src/routes/exchange.ts b/apps/api/src/routes/exchange.ts
index 7352b41c4..2fccecd0e 100644
--- a/apps/api/src/routes/exchange.ts
+++ b/apps/api/src/routes/exchange.ts
@@ -7,9 +7,11 @@ import {
acceptTermsSchema,
} from "../services/alexandria/terms";
import express, { Request, Response } from "express";
-import { Agent, fetch } from "undici";
+import { Agent, fetch, type Response as UpstreamResponse } from "undici";
+import type { Logger } from "winston";
import { config } from "../config";
import { recordAlexandriaActivity } from "../lib/alexandria-activity";
+import { teamRecommendationSignals } from "../services/alexandria/recommendation-signals";
import { logger as rootLogger } from "../lib/logger";
import type { RequestWithAuth } from "../controllers/v1/types";
import { RateLimiterMode } from "../types";
@@ -60,6 +62,43 @@ function upstreamBase(): string | null {
return config.FIRE_EXCHANGE_URL.replace(/\/+$/, "");
}
+function forwardedRequestHeaders(req: Request): Record<string, string> {
+ return Object.fromEntries(
+ FORWARDED_REQUEST_HEADERS.flatMap(h => {
+ const value = req.headers[h];
+ return typeof value === "string" ? [[h, value]] : [];
+ }),
+ );
+}
+
+function sendUpstream(res: Response, upstream: UpstreamResponse, text: string) {
+ for (const h of FORWARDED_RESPONSE_HEADERS) {
+ const value = upstream.headers.get(h);
+ if (value) res.setHeader(h, value);
+ }
+
+ let body: unknown;
+ try {
+ body = text ? JSON.parse(text) : null;
+ } catch {
+ body = text;
+ }
+
+ if (body === null || typeof body === "string") {
+ return res.status(upstream.status).send(body ?? "");
+ }
+ return res.status(upstream.status).json(body);
+}
+
+function upstreamFailure(res: Response, logger: Logger, error: unknown) {
+ if (error instanceof DOMException && error.name === "TimeoutError") {
+ logger.error("Exchange proxy timed out");
+ return exchangeError(res, 504, "The request timed out.");
+ }
+ logger.error("Exchange proxy error", { error });
+ return exchangeError(res, 502, "The request could not be completed.");
+}
+
// Rebuilds the upstream path from validated, re-encoded segments so it always
// names the route that authorized the request.
function upstreamPath(req: Request): string | null {
@@ -130,12 +169,7 @@ function exchangeProxy(
const upstream = await fetch(base + path, {
method: req.method,
headers: {
- ...Object.fromEntries(
- FORWARDED_REQUEST_HEADERS.flatMap(h => {
- const value = req.headers[h];
- return typeof value === "string" ? [[h, value]] : [];
- }),
- ),
+ ...forwardedRequestHeaders(req),
...(hasBody ? { "content-type": "application/json" } : {}),
"x-exchange-team-id": authedReq.auth.team_id,
...(req.ip ? { "x-exchange-client-ip": req.ip } : {}),
@@ -152,38 +186,59 @@ function exchangeProxy(
dispatcher,
});
- for (const h of FORWARDED_RESPONSE_HEADERS) {
- const value = upstream.headers.get(h);
- if (value) res.setHeader(h, value);
- }
-
const text = await upstream.text();
// Awaited so feedback sent right after this response sees the window.
if (options.opensFeedbackWindow && upstream.ok) {
await recordAlexandriaActivity(authedReq.auth.team_id);
}
- let body: unknown;
- try {
- body = text ? JSON.parse(text) : null;
- } catch {
- body = text;
- }
-
- if (body === null || typeof body === "string") {
- return res.status(upstream.status).send(body ?? "");
- }
- return res.status(upstream.status).json(body);
+ return sendUpstream(res, upstream, text);
} catch (error: unknown) {
- if (error instanceof DOMException && error.name === "TimeoutError") {
- logger.error("Exchange proxy timed out");
- return exchangeError(res, 504, "The request timed out.");
- }
- logger.error("Exchange proxy error", { error });
- return exchangeError(res, 502, "The request could not be completed.");
+ return upstreamFailure(res, logger, error);
}
};
}
+const recommendationsDispatcher = dispatcherFor(DISCOVER_TIMEOUT_MS);
+
+// The team's own usage signals go to the Exchange, which owns the ranking. The team is always the
+// authenticated one, and search queries are read only when the caller asks for them.
+async function recommendationsController(req: Request, res: Response) {
+ const authedReq = req as RequestWithAuth<any, any, any>;
+ const teamId = authedReq.auth.team_id;
+ const logger = rootLogger.child({
+ module: "api/exchange",
+ method: req.method,
+ path: req.path,
+ teamId,
+ });
+
+ const base = upstreamBase();
+ if (!base) {
+ return exchangeError(res, 503, "This endpoint is not available.");
+ }
+
+ const signals = await teamRecommendationSignals(teamId, {
+ searches: req.query.searches === "true",
+ });
+
+ try {
+ const upstream = await fetch(base + "/v1/recommendations", {
+ method: "POST",
+ headers: {
+ ...forwardedRequestHeaders(req),
+ "content-type": "application/json",
+ "x-exchange-team-id": teamId,
+ },
+ body: JSON.stringify(signals),
+ signal: AbortSignal.timeout(DISCOVER_TIMEOUT_MS),
+ dispatcher: recommendationsDispatcher,
+ });
+ return sendUpstream(res, upstream, await upstream.text());
+ } catch (error: unknown) {
+ return upstreamFailure(res, logger, error);
+ }
+}
+
async function providerTermsAcceptController(req: Request, res: Response) {
const authedReq = req as RequestWithAuth<any, any, any>;
const orgId = orgIdFromAcuc(authedReq.acuc);
@@ -233,6 +288,14 @@ exchangeRouter.get(
),
);
+// Providers suggested from the team's own usage. A read like discover, but each call aggregates
+// the team's recent activity, so it stays on the flat Labs limit rather than discover's.
+exchangeRouter.get(
+ "/recommendations",
+ authMiddleware(RateLimiterMode.Labs),
+ wrap(recommendationsController),
+);
+
// These read-only discovery routes remain authenticated; they do not execute paid tools.
exchangeRouter.post(
"/skills/resolve",
diff --git a/apps/api/src/services/alexandria/recommendation-signals.ts b/apps/api/src/services/alexandria/recommendation-signals.ts
new file mode 100644
index 000000000..bac374b7f
--- /dev/null
+++ b/apps/api/src/services/alexandria/recommendation-signals.ts
@@ -0,0 +1,133 @@
+import { clickhouseClient } from "../../lib/clickhouse-client";
+import { logger as rootLogger } from "../../lib/logger";
+
+const WINDOW_DAYS = 30;
+/** With `timeout_overflow_mode: break` this returns what was aggregated so far, not an error. */
+const CLICKHOUSE_MAX_SECONDS = 2;
+const CLICKHOUSE_TIMEOUT_MS = 3_000;
+/** Per endpoint kind: enough hosts and queries to fill the Exchange's lookups. */
+const ROWS_PER_KIND = 200;
+const HOST_LIMIT = 100;
+const SEARCH_LIMIT = 50;
+const QUERY_MAX_LENGTH = 200;
+
+/** Kinds whose `target_hint` is the URL the request was about (the first one, for batches). */
+const URL_KINDS = ["scrape", "batch_scrape", "crawl", "map", "extract"];
+/** What is stored as `target_hint` for zero-data-retention requests, in place of the input. */
+const ZDR_TARGET_HINT = "<redacted due to zero data retention>";
+
+const HOSTNAME =
+ /^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/;
+
+/** The team's own activity on the other endpoints, in the shape `/v1/recommendations` takes. */
+interface RecommendationSignals {
+ hosts: { host: string; requests: number }[];
+ searches?: { query: string; requests: number }[];
+ kinds: Record<string, number>;
+}
+
+function normalizeHost(value: string): string | null {
+ const host = value
+ .trim()
+ .toLowerCase()
+ .replace(/\.$/, "")
+ .replace(/^www\./, "");
+ return host.length <= 253 && HOSTNAME.test(host) ? host : null;
+}
+
+/**
+ * One aggregated read over the team's `requests` for the usage window: hosts it scraped, crawled,
+ * mapped or extracted from, requests per kind and, only when `searches` is set, its search
+ * queries. Without `searches` no search text is selected; search requests still count per kind.
+ * Alexandria calls are left out because the Exchange already knows them per provider. Distinct
+ * ids are counted because a redelivered row coexists with the original until parts merge.
+ *
+ * Never fails: without ClickHouse, or on an error or timeout, the signals are empty and the
+ * Exchange ranks from Alexandria usage and the catalogue alone.
+ */
+export async function teamRecommendationSignals(
+ teamId: string,
+ options: { searches: boolean },
+): Promise<RecommendationSignals> {
+ const signals: RecommendationSignals = {
+ hosts: [],
+ ...(options.searches ? { searches: [] } : {}),
+ kinds: {},
+ };
+ if (clickhouseClient === null) return signals;
+
+ let rows: { kind: string; target: string | null; n: number | string }[];
+ try {
+ const result = await clickhouseClient.query({
+ query: `
+ SELECT kind, target, n FROM (
+ SELECT
+ kind,
+ multiIf(
+ kind IN {urlKinds: Array(String)}, lower(domainWithoutWWW(target_hint)),
+ ${options.searches ? "kind = 'search', trimBoth(target_hint)," : ""}
+ ''
+ ) AS target,
+ toUInt32(uniq(id)) AS n
+ FROM requests
+ WHERE team_id = {teamId: UUID}
+ AND created_at >= {since: DateTime64(3)}
+ AND kind != 'alexandria'
+ AND NOT startsWith(coalesce(target_hint, ''), 'alexandria:')
+ AND NOT startsWith(coalesce(target_hint, ''), {redacted: String})
+ GROUP BY kind, target
+ )
+ ORDER BY n DESC
+ LIMIT {rowsPerKind: UInt32} BY kind
+ `,
+ query_params: {
+ teamId,
+ since: new Date(Date.now() - WINDOW_DAYS * 24 * 60 * 60 * 1000)
+ .toISOString()
+ .replace("T", " ")
+ .replace(/Z$/, ""),
+ urlKinds: URL_KINDS,
+ redacted: ZDR_TARGET_HINT,
+ rowsPerKind: ROWS_PER_KIND,
+ },
+ format: "JSONEachRow",
+ clickhouse_settings: {
+ max_execution_time: CLICKHOUSE_MAX_SECONDS,
+ timeout_overflow_mode: "break",
+ },
+ abort_signal: AbortSignal.timeout(CLICKHOUSE_TIMEOUT_MS),
+ });
+ rows = await result.json();
+ } catch (error) {
+ rootLogger.warn("Recommendation signals unavailable", {
+ module: "alexandria/recommendation-signals",
+ teamId,
+ error: error instanceof Error ? error.message : String(error),
+ });
+ return signals;
+ }
+
+ const hosts = new Map<string, number>();
+ for (const { kind, target, n } of rows) {
+ const requests = Number(n);
+ if (!Number.isFinite(requests) || requests <= 0) continue;
+ signals.kinds[kind] = (signals.kinds[kind] ?? 0) + requests;
+ if (!target) continue;
+ if (kind === "search") {
+ if (signals.searches && signals.searches.length < SEARCH_LIMIT) {
+ signals.searches.push({
+ query: target.slice(0, QUERY_MAX_LENGTH),
+ requests,
+ });
+ }
+ continue;
+ }
+ const host = normalizeHost(target);
+ if (host) hosts.set(host, (hosts.get(host) ?? 0) + requests);
+ }
+ signals.hosts = [...hosts.entries()]
+ .sort((a, b) => b[1] - a[1])
+ .slice(0, HOST_LIMIT)
+ .map(([host, requests]) => ({ host, requests }));
+ return signals;
+}