Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): log research origin header
mogeryd9762ee97ba6
diff --git a/apps/api/src/__tests__/snips/v2/lib.ts b/apps/api/src/__tests__/snips/v2/lib.tsindex 4092620e2..68fa667f3 100644--- a/apps/api/src/__tests__/snips/v2/lib.ts+++ b/apps/api/src/__tests__/snips/v2/lib.ts@@ -642,6 +642,7 @@ export async function researchRaw(path: string,query: Record<string, string | number | boolean | string[]> | undefined,identity?: Identity,+ headers?: Record<string, string>,) {const req = request(TEST_API_URL).get(path)@@ -649,6 +650,11 @@ export async function researchRaw(if (identity) {req.set("Authorization", `Bearer ${identity.apiKey}`);}+ if (headers) {+ for (const [key, value] of Object.entries(headers)) {+ req.set(key, value);+ }+ }return query ? req.query(query) : req;}diff --git a/apps/api/src/__tests__/snips/v2/research.test.ts b/apps/api/src/__tests__/snips/v2/research.test.tsindex d6c585b33..05a6b3aeb 100644--- a/apps/api/src/__tests__/snips/v2/research.test.ts+++ b/apps/api/src/__tests__/snips/v2/research.test.ts@@ -1,6 +1,9 @@import { config } from "../../../config";import { describeIf, TEST_PRODUCTION } from "../lib";import { creditUsage, idmux, researchRaw } from "./lib";+import { and, desc, eq } from "drizzle-orm";+import { db } from "../../../db/connection";+import * as schema from "../../../db/schema";const HAS_RESEARCH = !!config.RESEARCH_PROXY_URL;const KEYLESS_ENABLED =@@ -10,6 +13,20 @@ const KEYLESS_ENABLED =const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));const sleepForBilling = () => sleep(40000);+async function waitForSingleRow<T>(+ fetcher: () => Promise<T | null>,+ timeoutMs: number = 10000,+ intervalMs: number = 250,+): Promise<T | null> {+ const deadline = Date.now() + timeoutMs;+ while (Date.now() < deadline) {+ const row = await fetcher();+ if (row) return row;+ await sleep(intervalMs);+ }+ return null;+}+describeIf(HAS_RESEARCH)("Research API", () => {it("serves paper search from the canonical mount", async () => {const identity = await idmux({@@ -82,6 +99,68 @@ describeIf(HAS_RESEARCH)("Research API", () => {expect(res.body.success).toBe(false);});+ it("logs research origin from X-Origin and integration from query", async () => {+ if (!config.USE_DB_AUTHENTICATION) return;++ const identity = await idmux({+ name: "research/logs metadata",+ credits: 100,+ });+ const query = `research metadata ${Date.now()}`;++ const res = await researchRaw(+ "/v2/search/research/papers",+ { query, k: 1, integration: "_research_test" },+ identity,+ { "X-Origin": "mcp" },+ );++ expect(res.statusCode).toBe(200);+ expect(res.body.success).toBe(true);++ const requestLog = await waitForSingleRow<{+ origin: string | null;+ integration: string | null;+ }>(async () => {+ const data = await db+ .select({+ origin: schema.requests.origin,+ integration: schema.requests.integration,+ })+ .from(schema.requests)+ .where(+ and(+ eq(schema.requests.team_id, identity.teamId),+ eq(schema.requests.kind, "research_paper_search"),+ eq(schema.requests.target_hint, query),+ ),+ )+ .orderBy(desc(schema.requests.created_at))+ .limit(1);+ return data[0] ?? null;+ });++ expect(requestLog).not.toBeNull();+ expect(requestLog?.origin).toBe("mcp");+ expect(requestLog?.integration).toBe("_research_test");+ }, 120000);++ it("rejects invalid research integration values", async () => {+ const identity = await idmux({+ name: "research/invalid integration",+ credits: 100,+ });++ const res = await researchRaw(+ "/v2/search/research/papers",+ { query: "rag", integration: "invalid-integration" },+ identity,+ );++ expect(res.statusCode).toBe(400);+ expect(res.body.success).toBe(false);+ });+describeIf(KEYLESS_ENABLED)("keyless research", () => {it("permits keyless access on the canonical research index", async () => {const res = await researchRaw("/v2/search/research/papers", {diff --git a/apps/api/src/controllers/v2/research-proxy.ts b/apps/api/src/controllers/v2/research-proxy.tsindex c8f6a78a8..3eeeb7fe8 100644--- a/apps/api/src/controllers/v2/research-proxy.ts+++ b/apps/api/src/controllers/v2/research-proxy.ts@@ -17,6 +17,7 @@ import type {} from "../../services/logging/log_job";import type { RequestWithAuth } from "../v1/types";import { wrap } from "../../routes/shared";+import { integrationSchema } from "../../utils/integration";const TIMEOUT_MS = 120_000;const SEARCH_CREDITS_PER_TEN_RESULTS = 2;@@ -49,7 +50,7 @@ const dateSchema = zconst commonQuery = {origin: z.string().optional(),- integration: z.string().optional(),+ integration: integrationSchema.optional(),};const searchPapersSchema = z.strictObject({@@ -177,6 +178,16 @@ function resultCount(body: any): number {return Array.isArray(body?.results) ? body.results.length : 0;}+function firstHeaderValue(req: Request, key: string): string | undefined {+ const value = req.headers[key];+ if (Array.isArray(value)) return value[0];+ return typeof value === "string" ? value : undefined;+}++function requestOrigin(params: ResearchQueryParams, req: Request) {+ return params.origin ?? firstHeaderValue(req, "x-origin") ?? "api";+}+function creditsFor(config: ResearchEndpointConfig,body: any,@@ -266,7 +277,7 @@ function createResearchController(kind: endpoint.kind,api_version: "v2",team_id: authedReq.auth.team_id,- origin: params.origin ?? "api",+ origin: requestOrigin(params, req),integration: params.integration ?? null,target_hint: targetHint,zeroDataRetention: false,