Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Collect search feedback on results across all sources (#4109)
firecrawl-spring[bot]2ceda1b77b0f
diff --git a/apps/api/openapi.json b/apps/api/openapi.jsonindex 76809d30b..c254a4f1e 100644--- a/apps/api/openapi.json+++ b/apps/api/openapi.json@@ -3184,7 +3184,7 @@},"SearchFeedbackRequest": {"type": "object",- "description": "For 'good', include valuableSources. For 'partial', include valuableSources or missingContent. For 'bad', include missingContent or querySuggestions.",+ "description": "For 'good', include valuableSources or valuableResults. For 'partial', include valuableSources, valuableResults, or missingContent. For 'bad', include missingContent or querySuggestions.","properties": {"rating": {"type": "string",@@ -3208,6 +3208,30 @@"required": ["url"]}},+ "valuableResults": {+ "type": "array",+ "maxItems": 50,+ "description": "Results that were useful, each addressed by the `data` group it appears in and its 1-indexed position within that group. Results are returned grouped (`data.web`, `data.images`, `data.news`) and each group is numbered from 1 independently, so `source` is required. List every useful result β unlisted results are treated as not useful. Positions beyond what the group actually returned are ignored. The API stores these as stable search result document IDs for quantitative feedback.",+ "items": {+ "type": "object",+ "properties": {+ "source": {+ "type": "string",+ "enum": ["web", "images", "news"]+ },+ "position": {+ "type": "integer",+ "minimum": 1,+ "maximum": 100+ },+ "reason": {+ "type": "string",+ "maxLength": 1000+ }+ },+ "required": ["source", "position"]+ }+ },"missingContent": {"type": "array","maxItems": 20,@@ -3248,7 +3272,7 @@},{"type": "object",- "description": "Submit feedback for a v2 job. Include at least one substantive signal such as issues, note, valuableSources, missingContent, querySuggestions, url, or pageNumbers.",+ "description": "Submit feedback for a v2 job. Include at least one substantive signal such as issues, note, valuableSources, valuableResults for search, missingContent, querySuggestions, url, or pageNumbers.","properties": {"endpoint": {"type": "string",diff --git a/apps/api/src/__tests__/snips/v2/search-feedback.test.ts b/apps/api/src/__tests__/snips/v2/search-feedback.test.tsindex c607837ba..5fa245cfc 100644--- a/apps/api/src/__tests__/snips/v2/search-feedback.test.ts+++ b/apps/api/src/__tests__/snips/v2/search-feedback.test.ts@@ -484,4 +484,163 @@ describeIf(TEST_PRODUCTION)("Search feedback tests", () => {},90000,);++ // Positions are bounded and attributed from the feedback job, which is what+ // `lookupFeedbackJob` reads. Seeding it directly is how these tests control+ // the per-source counts without depending on what the live search returned.+ async function seedFeedbackJob(+ searchId: string,+ extra: {+ numResultsBySource?: Record<string, number>;+ resultCategories?: Record<string, Record<string, string>>;+ },+ ) {+ let job = await readFeedbackJob(searchId);+ for (let attempt = 0; attempt < 30 && !job; attempt++) {+ await new Promise(r => setTimeout(r, 1000));+ job = await readFeedbackJob(searchId);+ }+ expect(job).not.toBeNull();+ expect(+ await writeFeedbackJob({+ endpoint: "search",+ jobId: searchId,+ requestId: job!.requestId,+ teamId: identity.teamId,+ succeeded: job!.succeeded,+ creditsBilled: job!.creditsBilled,+ zeroDataRetention: job!.zeroDataRetention,+ ...extra,+ }),+ ).toBe(true);+ }++ async function feedbackMetadataFor(feedbackId: string) {+ const [row] = await db+ .select({ metadata: schema.search_feedback.metadata })+ .from(schema.search_feedback)+ .where(eq(schema.search_feedback.id, feedbackId))+ .limit(1);+ expect(row).toBeTruthy();+ return row.metadata as Record<string, unknown>;+ }++ it.concurrent(+ "bounds each group by its own result count",+ async () => {+ const raw = await searchRawFull(+ { query: "firecrawl grouped bounds", limit: 3 },+ identity,+ );+ expect(raw.statusCode).toBe(200);+ const searchId = raw.body.id;++ await seedFeedbackJob(searchId, {+ numResultsBySource: { web: 2, images: 0, news: 3 },+ });++ const result = await searchFeedback(+ searchId,+ {+ rating: "good",+ valuableResults: [+ { source: "web", position: 2 },+ // Past the web group, though the combined total leaves room.+ { source: "web", position: 3 },+ { source: "news", position: 3 },+ // images returned nothing, so no position in it is real.+ { source: "images", position: 1 },+ ],+ },+ identity,+ );+ expect(result.success).toBe(true);++ expect(await feedbackMetadataFor(result.feedbackId)).toEqual(+ expect.objectContaining({+ valuableResults: [+ { source: "web", position: 2 },+ { source: "news", position: 3 },+ ],+ valuableResultDocumentIds: [+ `search:${searchId}:web:1`,+ `search:${searchId}:news:2`,+ ],+ }),+ );+ },+ 90000,+ );++ it.concurrent(+ "records the serving vertical alongside the position",+ async () => {+ const raw = await searchRawFull(+ { query: "firecrawl vertical label", limit: 3 },+ identity,+ );+ expect(raw.statusCode).toBe(200);+ const searchId = raw.body.id;++ await seedFeedbackJob(searchId, {+ numResultsBySource: { web: 2, images: 0, news: 0 },+ resultCategories: { web: { "1": "developer" } },+ });++ const result = await searchFeedback(+ searchId,+ {+ rating: "good",+ valuableResults: [+ { source: "web", position: 1 },+ // Tagged by nothing, so it carries a position but no category.+ { source: "web", position: 2 },+ ],+ },+ identity,+ );+ expect(result.success).toBe(true);++ expect(await feedbackMetadataFor(result.feedbackId)).toEqual(+ expect.objectContaining({+ valuableResults: [+ { source: "web", position: 1, category: "developer" },+ { source: "web", position: 2 },+ ],+ }),+ );+ },+ 90000,+ );++ it.concurrent(+ "keeps positions unbounded when the job predates the counts",+ async () => {+ const raw = await searchRawFull(+ { query: "firecrawl legacy job", limit: 3 },+ identity,+ );+ expect(raw.statusCode).toBe(200);+ const searchId = raw.body.id;++ // No counts: a job written before the field existed. Unknown bound, so+ // the label is kept rather than discarded.+ await seedFeedbackJob(searchId, {});++ const result = await searchFeedback(+ searchId,+ { rating: "good", valuableResults: [{ source: "news", position: 2 }] },+ identity,+ );+ expect(result.success).toBe(true);++ expect(await feedbackMetadataFor(result.feedbackId)).toEqual(+ expect.objectContaining({+ valuableResults: [{ source: "news", position: 2 }],+ valuableResultDocumentIds: [`search:${searchId}:news:1`],+ }),+ );+ },+ 90000,+ );});diff --git a/apps/api/src/controllers/v2/feedback/feedback-store-bounds.test.ts b/apps/api/src/controllers/v2/feedback/feedback-store-bounds.test.tsnew file mode 100644index 000000000..e57431099--- /dev/null+++ b/apps/api/src/controllers/v2/feedback/feedback-store-bounds.test.ts@@ -0,0 +1,146 @@+import { beforeEach, describe, expect, it, vi } from "vitest";+import type { FeedbackJobRow, FeedbackRecordOptions } from "./internal-types";++const mocks = vi.hoisted(() => ({+ values: vi.fn().mockResolvedValue(undefined),+}));++vi.mock("../../../db/connection", () => ({+ db: { insert: () => ({ values: mocks.values }) },+ dbRr: { select: () => ({}) },+}));++vi.mock("../../../lib/feedback-job-store", () => ({+ readFeedbackJob: vi.fn(),+}));++import { insertFeedback } from "./feedback-store";++const searchId = "01933161-0000-7000-8000-000000000001";+const teamId = "01933161-0000-7000-8000-000000000002";++function searchJob(overrides: Partial<FeedbackJobRow> = {}): FeedbackJobRow {+ return {+ endpoint: "search",+ id: searchId,+ request_id: null,+ team_id: teamId,+ credits_cost: 2,+ created_at: new Date().toISOString(),+ is_successful: true,+ // The Bigtable feedback job carries no request body.+ options: null,+ ...overrides,+ };+}++function options(+ valuableResults: { source: string; position: number }[],+): FeedbackRecordOptions {+ return {+ endpoint: "search",+ jobId: searchId,+ feedback: { rating: "good", valuableResults } as any,+ source: "search_feedback",+ } as FeedbackRecordOptions;+}++async function recordedPositions(+ job: FeedbackJobRow,+ valuableResults: { source: string; position: number }[],+) {+ mocks.values.mockClear();+ await insertFeedback({+ feedbackId: "01933161-0000-7000-8000-00000000000f",+ options: options(valuableResults),+ job,+ dbTeamId: teamId,+ });+ const row = mocks.values.mock.calls[0]?.[0];+ return (row?.metadata as { valuableResults?: unknown[] })?.valuableResults;+}++describe("position bounds from the feedback job's per-source counts", () => {+ beforeEach(() => vi.clearAllMocks());++ it("bounds each group by its own count", async () => {+ const job = searchJob({+ num_results_by_source: { web: 5, images: 0, news: 3 },+ });++ expect(+ await recordedPositions(job, [+ { source: "web", position: 5 },+ // Past the web group, even though other groups have room.+ { source: "web", position: 6 },+ { source: "news", position: 3 },+ // images returned nothing, so no position in it is real.+ { source: "images", position: 1 },+ ]),+ ).toEqual([+ { source: "web", position: 5 },+ { source: "news", position: 3 },+ ]);+ });++ it("rejects every position for a group the counts omit", async () => {+ // A group absent from the counts returned nothing β there is no looser+ // bound to fall back to now that the counts are exact.+ const job = searchJob({ num_results_by_source: { web: 2 } });++ expect(+ await recordedPositions(job, [+ { source: "web", position: 1 },+ { source: "news", position: 1 },+ ]),+ ).toEqual([{ source: "web", position: 1 }]);+ });++ it("treats an empty count map as a search that returned nothing", async () => {+ const job = searchJob({ num_results_by_source: {} });++ expect(+ await recordedPositions(job, [{ source: "web", position: 1 }]),+ ).toBeUndefined();+ });++ it("leaves positions unbounded when the job predates the counts", async () => {+ // Absent is not empty. Feedback is only accepted within+ // SEARCH_FEEDBACK_MAX_AGE_SEC of the search, so this covers the couple of+ // minutes spanning the deploy and then closes on its own.+ const job = searchJob({ num_results_by_source: null });++ expect(+ await recordedPositions(job, [{ source: "news", position: 2 }]),+ ).toEqual([{ source: "news", position: 2 }]);+ });++ it("attributes the serving vertical when the job carries it", async () => {+ const job = searchJob({+ num_results_by_source: { web: 3, images: 0, news: 0 },+ result_categories: { web: { "1": "developer" } },+ });++ expect(+ await recordedPositions(job, [+ { source: "web", position: 1 },+ { source: "web", position: 2 },+ ]),+ ).toEqual([+ { source: "web", position: 1, category: "developer" },+ { source: "web", position: 2 },+ ]);+ });++ it("drops duplicate and non-positive positions", async () => {+ const job = searchJob({ num_results_by_source: { web: 3 } });++ expect(+ await recordedPositions(job, [+ { source: "web", position: 1 },+ { source: "web", position: 1 },+ { source: "web", position: 0 },+ ]),+ ).toEqual([{ source: "web", position: 1 }]);+ });+});diff --git a/apps/api/src/controllers/v2/feedback/feedback-store.ts b/apps/api/src/controllers/v2/feedback/feedback-store.tsindex d0f84580e..c985c3e2a 100644--- a/apps/api/src/controllers/v2/feedback/feedback-store.ts+++ b/apps/api/src/controllers/v2/feedback/feedback-store.ts@@ -8,10 +8,12 @@ import {} from "../../../lib/feedback-job-store";import { logger } from "../../../lib/logger";import { EndpointFeedbackEndpoint } from "../types";+import type { SearchResultType } from "../../../lib/entities";import {FeedbackJobRow,FeedbackRecordOptions,RefundPolicySnapshot,+ ValuableResultInput,} from "./internal-types";type DbError = { code?: string } & Record<string, unknown>;@@ -21,8 +23,192 @@ type ExistingFeedback = {credits_refunded: number | null;};+type ValuableResultDocument = {+ documentId: string;+ searchId: string;+ /** The `data` group the client addressed, as sent: web | images | news. */+ requestedSource: SearchResultType;+ /** Analytics spelling, as used in `documentId`. */+ resultType: SearchResultDocumentType;+ resultIndex: number;+ position: number;+ /**+ * Which vertical served this result, as the response tagged it β `developer`+ * for an index-served hit, `github` / `research` / `pdf` for a URL-derived+ * one. Absent when the result carried no category, or when the feedback job+ * predates `resultCategories`. This is the attribution half of the label:+ * `requestedSource` says where in `data` the result sat, `category` says who+ * answered.+ */+ category?: string;+ reason?: string;+ source: "position";+};++/**+ * Result type as recorded in analytics. The API groups image results under+ * `images` (matching `data.images`), but the ClickHouse `search_results` table+ * writes the singular `image`; document IDs use the analytics spelling so they+ * join directly on (search_id, result_type, result_index).+ */+type SearchResultDocumentType = "web" | "news" | "image";++const RESULT_DOCUMENT_TYPES: Record<+ SearchResultType,+ SearchResultDocumentType+> = {+ web: "web",+ news: "news",+ images: "image",+};++function searchFeedbackResultDocumentId(+ searchId: string,+ resultType: SearchResultDocumentType,+ resultIndex: number,+): string {+ return `search:${searchId}:${resultType}:${resultIndex}`;+}++/**+ * Number of results each `data` group returned, as the feedback job recorded+ * it. Returns null when the job predates the field or carries an unrecognised+ * shape β which is not the same as an empty object, meaning the search ran and+ * every group was empty.+ */+function resultCountsBySource(+ job: FeedbackJobRow,+): Partial<Record<SearchResultType, number>> | null {+ const counts = job.num_results_by_source;+ if (counts === null || counts === undefined || typeof counts !== "object") {+ return null;+ }++ // An object with no usable entries still counts as known: the search ran+ // and no group this API can address returned anything, which rejects every+ // position. Only a missing or non-object value is unknown.+ const entries = Object.entries(counts as Record<string, unknown>).filter(+ (entry): entry is [SearchResultType, number] =>+ entry[0] in RESULT_DOCUMENT_TYPES &&+ typeof entry[1] === "number" &&+ Number.isInteger(entry[1]) &&+ entry[1] >= 0,+ );+ return Object.fromEntries(entries);+}++/**+ * Vertical that served each position, as the feedback job recorded it. Returns+ * null when the job predates the field or carries an unrecognised shape, which+ * again is not the same as an empty object.+ */+function resultCategoriesBySource(+ job: FeedbackJobRow,+): Partial<Record<SearchResultType, Record<string, string>>> | null {+ const categories = job.result_categories;+ if (+ categories === null ||+ categories === undefined ||+ typeof categories !== "object"+ ) {+ return null;+ }++ const bySource: Partial<Record<SearchResultType, Record<string, string>>> =+ {};+ for (const [source, positions] of Object.entries(+ categories as Record<string, unknown>,+ )) {+ if (!(source in RESULT_DOCUMENT_TYPES)) continue;+ if (!positions || typeof positions !== "object") continue;++ const byPosition: Record<string, string> = {};+ for (const [position, category] of Object.entries(+ positions as Record<string, unknown>,+ )) {+ if (typeof category === "string" && category.length > 0) {+ byPosition[position] = category;+ }+ }+ bySource[source as SearchResultType] = byPosition;+ }++ return bySource;+}++/**+ * Highest 1-indexed position addressable in `data[source]`, or null when the+ * job carries no usable bound.+ *+ * The per-source counts bound every group exactly, so there is no looser+ * fallback to reach for: a group the counts do not mention returned nothing+ * and has no valid position. Null means the job predates the field β feedback+ * is only accepted within SEARCH_FEEDBACK_MAX_AGE_SEC of the search, so that+ * only covers searches run in the couple of minutes spanning the deploy, and+ * closes on its own.+ */+function maxResultPosition(+ job: FeedbackJobRow,+ source: SearchResultType,+): number | null {+ const counts = resultCountsBySource(job);+ if (counts === null) return null;+ return counts[source] ?? 0;+}++function valuableResultDocuments(+ job: FeedbackJobRow,+ results: ValuableResultInput[] | undefined,+): ValuableResultDocument[] {+ if (job.endpoint !== "search" || !results?.length) return [];++ // Rejects hallucinated positions, which would otherwise be stored as+ // false-positive relevance labels. Zero is a real bound: a group that+ // returned nothing has no valuable positions.+ const maxPositions = new Map<SearchResultType, number | null>();+ const seen = new Set<string>();+ const categories = resultCategoriesBySource(job);++ return results.flatMap(({ source, position, reason }) => {+ if (!(source in RESULT_DOCUMENT_TYPES)) return [];+ if (!Number.isInteger(position) || position <= 0) return [];++ if (!maxPositions.has(source)) {+ maxPositions.set(source, maxResultPosition(job, source));+ }+ const maxPosition = maxPositions.get(source) ?? null;+ if (maxPosition !== null && position > maxPosition) return [];++ const key = `${source}:${position}`;+ if (seen.has(key)) return [];+ seen.add(key);++ const resultType = RESULT_DOCUMENT_TYPES[source];+ const resultIndex = position - 1;+ const category = categories?.[source]?.[String(position)];+ return [+ {+ documentId: searchFeedbackResultDocumentId(+ job.id,+ resultType,+ resultIndex,+ ),+ searchId: job.id,+ requestedSource: source,+ resultType,+ resultIndex,+ position,+ ...(category ? { category } : {}),+ ...(reason ? { reason } : {}),+ source: "position" as const,+ },+ ];+ });+}+function feedbackMetadata(options: FeedbackRecordOptions,+ valuableResultDocs: ValuableResultDocument[],): Record<string, unknown> {return {...(options.feedback.metadata ?? {}),@@ -30,6 +216,20 @@ function feedbackMetadata(...(options.feedback.pageNumbers? { pageNumbers: options.feedback.pageNumbers }: {}),+ ...(valuableResultDocs.length > 0+ ? {+ // Echoed back in the vocabulary the client sent, not the analytics+ // spelling used in the document IDs.+ valuableResults: valuableResultDocs.map(doc => ({+ source: doc.requestedSource,+ position: doc.position,+ ...(doc.category ? { category: doc.category } : {}),+ })),+ valuableResultDocumentIds: valuableResultDocs.map(+ doc => doc.documentId,+ ),+ }+ : {}),};}@@ -72,6 +272,10 @@ export async function lookupFeedbackJob(feedback_deadline_ms: job.feedbackDeadlineMs,refund_class: job.refundClass,zero_data_retention: job.zeroDataRetention,+ num_results_by_source:+ endpoint === "search" ? (job.numResultsBySource ?? null) : null,+ result_categories:+ endpoint === "search" ? (job.resultCategories ?? null) : null,};}@@ -96,6 +300,10 @@ export async function insertFeedback(params: {apiKeyId?: number | null;}): Promise<DbError | null> {const { feedbackId, options, job, dbTeamId, apiKeyId } = params;+ const valuableResultDocs = valuableResultDocuments(+ job,+ options.feedback.valuableResults,+ );try {await db.insert(schema.search_feedback).values({id: feedbackId,@@ -110,10 +318,13 @@ export async function insertFeedback(params: {issue_types: options.feedback.issues ?? [],tags: options.feedback.tags ?? [],comment: options.feedback.note ?? null,- valuable_sources: options.feedback.valuableSources ?? [],+ valuable_sources: [+ ...(options.feedback.valuableSources ?? []),+ ...valuableResultDocs,+ ],missing_content: options.feedback.missingContent ?? [],query_suggestions: options.feedback.querySuggestions ?? null,- metadata: feedbackMetadata(options),+ metadata: feedbackMetadata(options, valuableResultDocs),job_status: job.is_successful === false ? "failed" : "completed",credits_billed: job.credits_cost ?? 0,credits_refunded: 0,diff --git a/apps/api/src/controllers/v2/feedback/internal-types.ts b/apps/api/src/controllers/v2/feedback/internal-types.tsindex b9b196c4c..8cfc509c9 100644--- a/apps/api/src/controllers/v2/feedback/internal-types.ts+++ b/apps/api/src/controllers/v2/feedback/internal-types.ts@@ -1,4 +1,5 @@import { logger as _logger } from "../../../lib/logger";+import type { SearchResultType } from "../../../lib/entities";import {EndpointFeedbackEndpoint,EndpointFeedbackErrorCode,@@ -9,12 +10,20 @@ import type { RefundClass } from "../../../lib/feedback-job-store";export type FeedbackRating = "good" | "partial" | "bad";+/** A valuable result addressed by its group and 1-indexed position in it. */+export type ValuableResultInput = {+ source: SearchResultType;+ position: number;+ reason?: string;+};+export type FeedbackInput = {rating: FeedbackRating;issues?: string[];tags?: string[];note?: string;valuableSources?: Array<{ url: string; reason?: string }>;+ valuableResults?: ValuableResultInput[];missingContent?: Array<{ topic: string; description?: string }>;querySuggestions?: string;url?: string;@@ -33,6 +42,19 @@ export type FeedbackJobRow = {created_at: string;is_successful: boolean | null;options: unknown;+ /**+ * Per-source result counts (search only), e.g. `{"web":3,"images":0}`, read+ * from the Bigtable feedback job. Null on jobs written before the field+ * existed; `{}` means the search ran and every group was empty.+ */+ num_results_by_source?: unknown;+ /**+ * Which vertical served each position (search only), e.g.+ * `{"web":{"1":"developer"}}`. Sparse: untagged results and untagged groups+ * are absent. Null on jobs written before the field existed and on+ * zero-data-retention searches.+ */+ result_categories?: unknown;feedback_deadline_ms?: number;refund_class?: RefundClass;zero_data_retention?: boolean;diff --git a/apps/api/src/controllers/v2/feedback/record-options.test.ts b/apps/api/src/controllers/v2/feedback/record-options.test.tsindex 477dc5c52..d458f9b28 100644--- a/apps/api/src/controllers/v2/feedback/record-options.test.ts+++ b/apps/api/src/controllers/v2/feedback/record-options.test.ts@@ -97,6 +97,49 @@ describe("feedback schema", () => {}).success,).toBe(true);+ expect(+ searchFeedbackSchema.safeParse({+ rating: "good",+ valuableResults: [+ { source: "web", position: 1 },+ { source: "news", position: 2, reason: "Broke the story." },+ { source: "images", position: 1 },+ ],+ }).success,+ ).toBe(true);++ expect(+ endpointFeedbackSchema.safeParse({+ endpoint: "search",+ jobId: "01933161-0000-7000-8000-000000000001",+ rating: "good",+ valuableResults: [{ source: "web", position: 1 }],+ }).success,+ ).toBe(true);++ // `source` is required β a bare position is ambiguous across groups.+ expect(+ searchFeedbackSchema.safeParse({+ rating: "good",+ valuableResults: [{ position: 1 }],+ }).success,+ ).toBe(false);++ expect(+ searchFeedbackSchema.safeParse({+ rating: "good",+ valuableResults: [{ source: "video", position: 1 }],+ }).success,+ ).toBe(false);++ // The old flat form is no longer accepted.+ expect(+ searchFeedbackSchema.safeParse({+ rating: "good",+ valuableResultPositions: [1, 2],+ }).success,+ ).toBe(false);+expect(endpointFeedbackSchema.safeParse({endpoint: "search",@@ -114,5 +157,35 @@ describe("feedback schema", () => {note: "Generic endpoint feedback remains valid with a note.",}).success,).toBe(true);++ expect(+ endpointFeedbackSchema.safeParse({+ endpoint: "map",+ jobId: "01933161-0000-7000-8000-000000000001",+ rating: "good",+ valuableResults: [{ source: "web", position: 1 }],+ }).success,+ ).toBe(false);++ expect(+ searchFeedbackSchema.safeParse({+ rating: "good",+ valuableResults: [{ source: "web", position: 0 }],+ }).success,+ ).toBe(false);++ expect(+ searchFeedbackSchema.safeParse({+ rating: "good",+ valuableResults: [{ source: "web", position: 101 }],+ }).success,+ ).toBe(false);++ expect(+ searchFeedbackSchema.safeParse({+ rating: "good",+ valuableResults: [{ source: "web", position: 100 }],+ }).success,+ ).toBe(true);});});diff --git a/apps/api/src/controllers/v2/feedback/request-input.ts b/apps/api/src/controllers/v2/feedback/request-input.tsindex d05d2d09a..3e3370452 100644--- a/apps/api/src/controllers/v2/feedback/request-input.ts+++ b/apps/api/src/controllers/v2/feedback/request-input.ts@@ -7,6 +7,7 @@ export function toFeedbackInput(return {rating: body.rating,valuableSources: body.valuableSources,+ valuableResults: body.valuableResults,missingContent: body.missingContent,querySuggestions: body.querySuggestions,origin: body.origin,diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 568bdf28d..141bc9fb7 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -539,6 +539,8 @@ async function searchControllerInner(error: undefined,results: result.response as any,num_results: result.totalResultsCount,+ num_results_by_source: result.resultCountsBySource,+ result_categories: result.resultCategories,time_taken: timeTakenInSeconds,team_id: req.auth.team_id,options: req.body,diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 2c9e875e0..c051c9b9a 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -2845,22 +2845,40 @@ const missingContentEntrySchema = z.strictObject({description: z.string().trim().max(2000).optional(),});+/**+ * A single valuable search result, addressed by the group it appears in and its+ * 1-indexed position within that group. Results are returned grouped+ * (`data.web`, `data.images`, `data.news`), each group sliced independently to+ * `limit`, so a bare position is ambiguous once more than one source is in play.+ */+const valuableResultSchema = z.strictObject({+ source: z.enum(["web", "images", "news"]),+ // 100 is the max search `limit`, and each group is capped at `limit`+ // independently, so positions beyond it can only be hallucinated.+ position: z.number().int().positive().lte(100),+ reason: z.string().trim().max(1000).optional(),+});++const valuableResultsSchema = z.array(valuableResultSchema).max(50);+function hasSubstantiveSearchFeedback(data: {rating: "good" | "bad" | "partial";valuableSources?: unknown[];+ valuableResults?: unknown[];missingContent?: unknown[];querySuggestions?: string;}): boolean {const hasSources = (data.valuableSources?.length ?? 0) > 0;+ const hasResults = (data.valuableResults?.length ?? 0) > 0;const hasMissing = (data.missingContent?.length ?? 0) > 0;const hasSuggestions =!!data.querySuggestions && data.querySuggestions.length > 0;switch (data.rating) {case "good":- return hasSources;+ return hasSources || hasResults;case "partial":- return hasSources || hasMissing;+ return hasSources || hasResults || hasMissing;case "bad":return hasMissing || hasSuggestions;}@@ -2880,6 +2898,7 @@ export const searchFeedbackSchema = z).max(50).optional(),+ valuableResults: valuableResultsSchema.optional(),missingContent: z.array(missingContentEntrySchema).max(20).optional(),querySuggestions: z.string().trim().max(2000).optional(),origin: z.string().optional().prefault("api"),@@ -2887,7 +2906,7 @@ export const searchFeedbackSchema = z}).refine(data => hasSubstantiveSearchFeedback(data), {message:- "Feedback must be substantive. 'good' requires at least one valuableSources entry; 'partial' requires valuableSources or at least one missingContent entry; 'bad' requires at least one missingContent entry or querySuggestions.",+ "Feedback must be substantive. 'good' requires valuableSources or valuableResults; 'partial' requires valuableSources, valuableResults, or at least one missingContent entry; 'bad' requires at least one missingContent entry or querySuggestions.",});export type SearchFeedbackRequest = z.infer<typeof searchFeedbackSchema>;@@ -2968,6 +2987,7 @@ export const endpointFeedbackSchema = z).max(50).optional(),+ valuableResults: valuableResultsSchema.optional(),missingContent: z.array(missingContentEntrySchema).max(50).optional(),querySuggestions: z.string().trim().max(2000).optional(),url: searchFeedbackUrlSchema.optional(),@@ -2983,6 +3003,7 @@ export const endpointFeedbackSchema = z(data.tags?.length ?? 0) > 0 ||(data.note?.length ?? 0) > 0 ||(data.valuableSources?.length ?? 0) > 0 ||+ (data.valuableResults?.length ?? 0) > 0 ||(data.missingContent?.length ?? 0) > 0 ||!!data.querySuggestions ||!!data.url ||@@ -2991,14 +3012,21 @@ export const endpointFeedbackSchema = z},{message:- "Feedback must include at least one substantive signal: issues, note, sources, missingContent, querySuggestions, url, or pageNumbers.",+ "Feedback must include at least one substantive signal: issues, note, sources, valuableResults, missingContent, querySuggestions, url, or pageNumbers.",+ },+ )+ .refine(+ data =>+ data.endpoint === "search" || (data.valuableResults?.length ?? 0) === 0,+ {+ message: "valuableResults is only supported for search feedback.",},).refine(data => data.endpoint !== "search" || hasSubstantiveSearchFeedback(data),{message:- "Search feedback must be substantive. 'good' requires at least one valuableSources entry; 'partial' requires valuableSources or at least one missingContent entry; 'bad' requires at least one missingContent entry or querySuggestions.",+ "Search feedback must be substantive. 'good' requires valuableSources or valuableResults; 'partial' requires valuableSources, valuableResults, or at least one missingContent entry; 'bad' requires at least one missingContent entry or querySuggestions.",},);diff --git a/apps/api/src/lib/__tests__/search-result-categories.test.ts b/apps/api/src/lib/__tests__/search-result-categories.test.tsnew file mode 100644index 000000000..56ef94303--- /dev/null+++ b/apps/api/src/lib/__tests__/search-result-categories.test.ts@@ -0,0 +1,115 @@+import { describe, it, expect } from "vitest";+import {+ collectSearchResultCategories,+ countSearchResultsBySource,+ type SearchV2Response,+} from "../entities";++const web = (url: string, category?: string) => ({+ url,+ title: url,+ description: "",+ ...(category ? { category } : {}),+});++describe("collectSearchResultCategories", () => {+ it("keys the serving vertical by group and 1-indexed position", () => {+ const response: SearchV2Response = {+ web: [+ web("https://github.com/firecrawl/firecrawl", "github"),+ web("https://example.com/"),+ web("https://arxiv.org/abs/1", "research"),+ ],+ };++ expect(collectSearchResultCategories(response)).toEqual({+ web: { "1": "github", "3": "research" },+ });+ });++ it("tags every developer hit, which is the whole web group", () => {+ // The developer category is exclusive, so its hits ARE data.web β+ // renumbered from 1 after threat filtering.+ const response: SearchV2Response = {+ web: [+ web("https://github.com/firecrawl/firecrawl/issues/1", "developer"),+ web("https://docs.firecrawl.dev/", "developer"),+ ],+ };++ expect(collectSearchResultCategories(response)).toEqual({+ web: { "1": "developer", "2": "developer" },+ });+ });++ it("distinguishes an untagged search from an absent one", () => {+ // `{}` is a claim: the search ran and nothing carried a category. NULL in+ // the column is the separate claim that the row cannot answer at all.+ expect(+ collectSearchResultCategories({ web: [web("https://example.com/")] }),+ ).toEqual({});+ expect(collectSearchResultCategories({})).toEqual({});+ });++ it("omits groups with no tagged results rather than emitting empty maps", () => {+ const response: SearchV2Response = {+ web: [web("https://github.com/firecrawl/cli", "github")],+ news: [{ url: "https://news.example/a", title: "A" }],+ images: [{ url: "https://img.example/a.png", title: "A" }],+ };++ expect(collectSearchResultCategories(response)).toEqual({+ web: { "1": "github" },+ });+ });++ it("keys news positions independently of web", () => {+ const response: SearchV2Response = {+ web: [web("https://example.com/")],+ news: [+ { url: "https://news.example/a", title: "A", category: "research" },+ ],+ };++ expect(collectSearchResultCategories(response)).toEqual({+ news: { "1": "research" },+ });+ });++ it("ignores non-string and empty categories", () => {+ const response = {+ web: [+ { url: "https://a.example/", title: "", description: "", category: "" },+ {+ url: "https://b.example/",+ title: "",+ description: "",+ category: 7 as unknown as string,+ },+ web("https://c.example/", "pdf"),+ ],+ } as SearchV2Response;++ expect(collectSearchResultCategories(response)).toEqual({+ web: { "3": "pdf" },+ });+ });++ it("positions line up with the counts feedback bounds against", () => {+ const response: SearchV2Response = {+ web: [web("https://a.example/", "developer"), web("https://b.example/")],+ news: [{ url: "https://news.example/a", title: "A" }],+ };++ const counts = countSearchResultsBySource(response);+ const categories = collectSearchResultCategories(response);++ for (const [source, byPosition] of Object.entries(categories)) {+ for (const position of Object.keys(byPosition)) {+ expect(Number(position)).toBeLessThanOrEqual(+ counts[source as keyof typeof counts],+ );+ }+ }+ });+});diff --git a/apps/api/src/lib/entities.ts b/apps/api/src/lib/entities.tsindex 440bf134f..179f79f30 100644--- a/apps/api/src/lib/entities.ts+++ b/apps/api/src/lib/entities.ts@@ -170,6 +170,73 @@ export interface SearchV2Response {news?: NewsSearchResult[];}+/**+ * Length of each `SearchV2Response` group as returned to the client. Every+ * group is present and non-negative β a source that was not requested, or that+ * returned nothing, counts 0. Persisted per search so position-based feedback+ * can be bounded exactly per source rather than against a combined total.+ */+export type SearchResultCountsBySource = Record<SearchResultType, number>;++export function countSearchResultsBySource(+ response: SearchV2Response,+): SearchResultCountsBySource {+ return {+ web: response.web?.length ?? 0,+ images: response.images?.length ?? 0,+ news: response.news?.length ?? 0,+ };+}++/**+ * Which vertical produced each result, keyed by group and then by the result's+ * 1-indexed position in that group: `{"web":{"1":"developer","3":"github"}}`.+ *+ * Sparse on purpose. Results carry a `category` only when one applies β the+ * developer index stamps `developer` on every hit it serves (see+ * `search/developer.ts`), and `github` / `research` / `pdf` are derived per URL+ * in `search/execute.ts` β so untagged results are simply absent, as are groups+ * with no tagged results at all. An empty object therefore means "nothing was+ * tagged", which is a different claim from a NULL column ("this row predates+ * the column").+ *+ * This is the only place the serving vertical survives the response. `source`+ * says which `data` group a result sits in, which is addressing; `category`+ * says who answered, which is attribution. They are the same value today only+ * because the developer category is schema-enforced as exclusive and lands in+ * `data.web`; blend a vertical into ordinary web results and the two come+ * apart. Persisted per search so position-based feedback can resolve the+ * vertical behind a `(source, position)` pair it is handed later.+ */+export type SearchResultCategoriesBySource = Partial<+ Record<SearchResultType, Record<string, string>>+>;++export function collectSearchResultCategories(+ response: SearchV2Response,+): SearchResultCategoriesBySource {+ const collected: SearchResultCategoriesBySource = {};++ for (const source of ["web", "images", "news"] as const) {+ const results = response[source];+ if (!results?.length) continue;++ const categories: Record<string, string> = {};+ results.forEach((result, index) => {+ const category = (result as { category?: unknown }).category;+ if (typeof category === "string" && category.length > 0) {+ categories[String(index + 1)] = category;+ }+ });++ if (Object.keys(categories).length > 0) {+ collected[source] = categories;+ }+ }++ return collected;+}+export interface ScrapeActionContent {url: string;html: string;diff --git a/apps/api/src/lib/feedback-job-store.ts b/apps/api/src/lib/feedback-job-store.tsindex 06b542b09..448141d0d 100644--- a/apps/api/src/lib/feedback-job-store.ts+++ b/apps/api/src/lib/feedback-job-store.ts@@ -39,6 +39,23 @@ type FeedbackJob = {succeeded: boolean;creditsBilled: number;zeroDataRetention: boolean;+ /**+ * Search only. How many results each group returned, e.g.+ * `{"web":3,"images":0,"news":10}`, and which vertical served each position,+ * e.g. `{"web":{"1":"developer"}}`.+ *+ * Position feedback addresses a result as `{source, position}`, 1-indexed+ * within its group, so bounding a position needs the length of that group+ * and nothing else. They live here rather than beside the search row+ * because this is the record the feedback window is enforced from, and it+ * is the only one feedback reads.+ *+ * Absent on records written before these fields existed, on non-search+ * jobs, and under zero data retention. Absent is not empty: `{}` means the+ * search ran and every group was empty, which rejects every position.+ */+ numResultsBySource?: Record<string, number>;+ resultCategories?: Record<string, Record<string, string>>;};function parseFeedbackJob(value: Buffer | string): FeedbackJob {@@ -132,6 +149,8 @@ type FeedbackJobBase = {creditsBilled: number;zeroDataRetention: boolean;completedAt?: Date;+ numResultsBySource?: Record<string, number>;+ resultCategories?: Record<string, Record<string, string>>;};type FeedbackJobWrite = FeedbackJobBase &@@ -187,6 +206,19 @@ export async function writeFeedbackJob(succeeded: params.succeeded,creditsBilled: params.creditsBilled,zeroDataRetention: params.zeroDataRetention,+ // Both describe the response, so zero data retention redacts them+ // with everything else. Feedback on a ZDR job is dropped before the+ // bounds are read, so nothing needs them for those rows.+ ...(params.zeroDataRetention+ ? {}+ : {+ ...(params.numResultsBySource+ ? { numResultsBySource: params.numResultsBySource }+ : {}),+ ...(params.resultCategories+ ? { resultCategories: params.resultCategories }+ : {}),+ }),}),);const table = await getBigtableTable(tableId);diff --git a/apps/api/src/search/execute.test.ts b/apps/api/src/search/execute.test.tsindex 9090c21bb..3a1e558c2 100644--- a/apps/api/src/search/execute.test.ts+++ b/apps/api/src/search/execute.test.ts@@ -219,6 +219,9 @@ describe("executeSearch developer category", () => {expect(result.response).toEqual({ web: [developerResult] });expect(result.response).not.toHaveProperty("developer");expect(result.indexResultsCount).toBe(1);+ // The vertical that served the result survives the response as a label+ // keyed by the position the client sees.+ expect(result.resultCategories).toEqual({ web: { "1": "developer" } });});it("filters blocked developer results via threat protection and renumbers", async () => {@@ -263,6 +266,47 @@ describe("executeSearch developer category", () => {["https://ok.example/a", 1],["https://ok.example/c", 2],]);+ // Keyed by the renumbered positions, not the upstream ones β position 2 is+ // the surviving third result.+ expect(result.resultCategories).toEqual({+ web: { "1": "developer", "2": "developer" },+ });+ });++ it("returns an empty group and no categories when every result is blocked", async () => {+ mocks.searchDeveloperCategory.mockResolvedValue([+ {+ url: "https://blocked.example/a",+ title: "A",+ description: "",+ position: 1,+ category: "developer",+ },+ {+ url: "https://blocked.example/b",+ title: "B",+ description: "",+ position: 2,+ category: "developer",+ },+ ]);+ mocks.checkUrlsAgainstThreatPolicy.mockResolvedValue({+ decisionsByUrl: new Map([+ ["https://blocked.example/a", { allowed: false }],+ ["https://blocked.example/b", { allowed: false }],+ ]),+ });++ const result = await executeSearch(+ options([{ type: "developer" }]),+ { ...context, threatProtectionPolicy: { mode: "block" } } as any,+ logger,+ );++ expect(result.response.web ?? []).toEqual([]);+ // An empty map, not absent: the search ran and tagged nothing, which+ // feedback must tell apart from a row that predates the column.+ expect(result.resultCategories).toEqual({});});it("rejects developer combined with other categories at the schema", () => {diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.tsindex 257fcace3..903532aad 100644--- a/apps/api/src/search/execute.ts+++ b/apps/api/src/search/execute.ts@@ -1,7 +1,14 @@import type { Logger } from "winston";import { discoverTools, isAlexandriaSource } from "./alexandria";import { search } from "./v2";-import { SearchV2Response, SearchResultType } from "../lib/entities";+import {+ SearchV2Response,+ SearchResultType,+ SearchResultCountsBySource,+ SearchResultCategoriesBySource,+ countSearchResultsBySource,+ collectSearchResultCategories,+} from "../lib/entities";import {buildSearchQuery,getCategoryFromUrl,@@ -77,6 +84,8 @@ interface SearchExecuteResult {toolsUnavailable?: boolean;response: SearchV2Response;totalResultsCount: number;+ resultCountsBySource: SearchResultCountsBySource;+ resultCategories: SearchResultCategoriesBySource;indexResultsCount: number;searchCredits: number;scrapeCredits: number;@@ -453,6 +462,13 @@ export async function executeSearch(toolsWarning,toolsUnavailable,totalResultsCount,+ // Counted from the final response β after scraping and highlights β so it+ // matches exactly what the client can address by position.+ resultCountsBySource: countSearchResultsBySource(searchResponse),+ // Read from the same final response, so a category is keyed by the position+ // the client sees. Index hits replace the web group above and are+ // renumbered there, so their positions here are the renumbered ones.+ resultCategories: collectSearchResultCategories(searchResponse),indexResultsCount,searchCredits,scrapeCredits,diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex 6cfabebe6..62243c3c0 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -16,6 +16,10 @@ import {import { hasFormatOfType } from "../../lib/format-utils";import { keylessTeamUuid } from "../../lib/keyless";import type { Document, ScrapeOptions } from "../../controllers/v2/types";+import type {+ SearchResultCountsBySource,+ SearchResultCategoriesBySource,+} from "../../lib/entities";import type { CostTracking } from "../../lib/cost-tracking";import type { Logger } from "winston";import { saveExtractResult } from "../../lib/extract/extract-redis";@@ -983,6 +987,8 @@ export type LoggedSearch = {is_successful: boolean;error?: string;num_results: number;+ num_results_by_source?: SearchResultCountsBySource;+ result_categories?: SearchResultCategoriesBySource;results: any;zeroDataRetention: boolean;};@@ -1028,6 +1034,18 @@ async function logSearchInternal(search: LoggedSearch) {succeeded: search.is_successful,creditsBilled: search.credits_cost,zeroDataRetention: search.zeroDataRetention,+ // Position feedback bounds each `{source, position}` against the group+ // it addresses, and attributes it to the vertical that served it. This+ // is the record feedback reads, and the one the feedback window is+ // enforced from, so the counts travel with it. `writeFeedbackJob`+ // redacts both under zero data retention.+ //+ // Only v2 search reports counts. A v0/v1 search returns a flat list+ // whose positions do not map to `{source, position}`, so it records `{}`+ // and every position is rejected. Leaving it absent would read as a job+ // that predates the field and accept any position unbounded.+ numResultsBySource: search.num_results_by_source ?? {},+ resultCategories: search.result_categories,},logger,);@@ -1048,6 +1066,18 @@ async function logSearchInternal(search: LoggedSearch) {is_successful: search.is_successful,error: search.zeroDataRetention ? null : (search.error ?? null),num_results: search.num_results,+ // Both are response-derived and both are redacted under zero data+ // retention: the per-source split says how the result set was composed+ // ({"web":3,"news":10} characterises the query's coverage in a way the+ // combined total does not), and the category map says which vertical+ // served each position. Feedback on a ZDR search is not persisted+ // anyway, so neither column has a reader for those rows.+ num_results_by_source: search.zeroDataRetention+ ? null+ : (search.num_results_by_source ?? null),+ result_categories: search.zeroDataRetention+ ? null+ : (search.result_categories ?? null),time_taken: search.time_taken,},logger,