Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): log each Alexandria call's provenance (#5657)
Nicolas40c8ff95d719
diff --git a/apps/api/src/controllers/v2/scrape-alexandria.test.ts b/apps/api/src/controllers/v2/scrape-alexandria.test.tsindex cc744b011..040790233 100644--- a/apps/api/src/controllers/v2/scrape-alexandria.test.ts+++ b/apps/api/src/controllers/v2/scrape-alexandria.test.ts@@ -373,6 +373,42 @@ it("executes with supplied arguments but stores only tool identifiers in activit});});+it("logs each call's provenance and keeps it out of the response", async () => {+ mocks.retrieve.mockResolvedValue(+ result([+ { ...call, provenance: "workflows_programs", creditsCost: 0, data: {} },+ ]),+ );+ const response = await request(app)+ .post("/v2/scrape")+ .send({ alexandria: [call] });+ expect(response.status).toBe(200);+ expect(response.body.data.alexandria[0]).not.toHaveProperty("provenance");+ await vi.waitFor(() => expect(mocks.scrapeLog).toHaveBeenCalledTimes(1));+ expect(mocks.scrapeLog.mock.calls[0][0].options.alexandria[0]).toEqual(+ expect.objectContaining({ provenance: "workflows_programs" }),+ );+ const legacy = await request(app).post("/exchange/retrieve").send(call);+ expect(legacy.status).toBe(200);+ expect(legacy.body).not.toHaveProperty("provenance");+});++it("drops an unknown or null provenance instead of failing the call", async () => {+ mocks.retrieve.mockResolvedValue(+ result([+ { ...call, provenance: "elsewhere", creditsCost: 0, data: {} },+ { ...call, provenance: null, creditsCost: 0, data: {} },+ ]),+ );+ const response = await request(app)+ .post("/v2/scrape")+ .send({ alexandria: [call, call] });+ expect(response.status).toBe(200);+ await vi.waitFor(() => expect(mocks.scrapeLog).toHaveBeenCalledTimes(1));+ for (const logged of mocks.scrapeLog.mock.calls[0][0].options.alexandria)+ expect(logged).not.toHaveProperty("provenance");+});+it("logs each call's own outcome code and credits in request order", async () => {const other = { provider: "sec", capability: "filings/search", options: {} };const answer = result([diff --git a/apps/api/src/controllers/v2/scrape-alexandria.ts b/apps/api/src/controllers/v2/scrape-alexandria.tsindex fa26123d9..c01d78c50 100644--- a/apps/api/src/controllers/v2/scrape-alexandria.ts+++ b/apps/api/src/controllers/v2/scrape-alexandria.ts@@ -285,6 +285,9 @@ export async function providerScrapeController(? {code: served.data.results[i].error?.code ?? null,creditsCost: served.data.results[i].creditsCost ?? 0,+ ...(served.data.results[i].provenance+ ? { provenance: served.data.results[i].provenance }+ : {}),}: {}),...(isFindTools({ provider, capability })@@ -347,7 +350,13 @@ export async function providerScrapeController(res.setHeader("Retry-After", String(refused.retryAfterSeconds));return res.status(result.status).json(result.body);}- const parsed = answerSchema.parse(result.body);+ const exchangeAnswer = answerSchema.parse(result.body);+ const parsed = {+ ...exchangeAnswer,+ results: exchangeAnswer.results.map(+ ({ provenance: _provenance, ...item }) => item,+ ),+ };const answer = zeroDataRetention? parsed: {diff --git a/apps/api/src/services/alexandria/contracts.ts b/apps/api/src/services/alexandria/contracts.tsindex 2f0770cc7..078caad50 100644--- a/apps/api/src/services/alexandria/contracts.ts+++ b/apps/api/src/services/alexandria/contracts.ts@@ -20,10 +20,17 @@ export type ProviderCall = z.infer<typeof callSchema>;const credits = z.number().int().nonnegative().safe();const identity = { provider: z.string(), capability: z.string() };+// Where Exchange says the provider is built; logged per call, never returned to the caller.+// An unknown or null value is dropped rather than failing a call that already ran.+const provenance = z+ .enum(["connector", "exchange", "workflows_programs"])+ .nullish()+ .catch(undefined);const resultSchema = z.union([z.object({...identity,+ provenance,creditsCost: credits,data: z.unknown().refine(value => value !== undefined),error: z.never().optional(),@@ -33,6 +40,7 @@ const resultSchema = z.union([.object({provider: z.string().optional(),capability: z.string().optional(),+ provenance,creditsCost: z.literal(0).optional(),error: z.object({