Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

Collect search feedback on results across all sources (#4109)

firecrawl-spring[bot]2ceda1b77b0f
diff --git a/apps/api/openapi.json b/apps/api/openapi.json
index 76809d30b..c254a4f1e 100644
--- a/apps/api/openapi.json
+++ b/apps/api/openapi.json
@@ -3184,7 +3184,7 @@
},
"SearchFeedbackRequest": {
"type": "object",
- "description": "For 'good', include valuableSources. For 'partial', include valuableSources or missingContent. For 'bad', include missingContent or querySuggestions.",
+ "description": "For 'good', include valuableSources or valuableResults. For 'partial', include valuableSources, valuableResults, or missingContent. For 'bad', include missingContent or querySuggestions.",
"properties": {
"rating": {
"type": "string",
@@ -3208,6 +3208,30 @@
"required": ["url"]
}
},
+ "valuableResults": {
+ "type": "array",
+ "maxItems": 50,
+ "description": "Results that were useful, each addressed by the `data` group it appears in and its 1-indexed position within that group. Results are returned grouped (`data.web`, `data.images`, `data.news`) and each group is numbered from 1 independently, so `source` is required. List every useful result β€” unlisted results are treated as not useful. Positions beyond what the group actually returned are ignored. The API stores these as stable search result document IDs for quantitative feedback.",
+ "items": {
+ "type": "object",
+ "properties": {
+ "source": {
+ "type": "string",
+ "enum": ["web", "images", "news"]
+ },
+ "position": {
+ "type": "integer",
+ "minimum": 1,
+ "maximum": 100
+ },
+ "reason": {
+ "type": "string",
+ "maxLength": 1000
+ }
+ },
+ "required": ["source", "position"]
+ }
+ },
"missingContent": {
"type": "array",
"maxItems": 20,
@@ -3248,7 +3272,7 @@
},
{
"type": "object",
- "description": "Submit feedback for a v2 job. Include at least one substantive signal such as issues, note, valuableSources, missingContent, querySuggestions, url, or pageNumbers.",
+ "description": "Submit feedback for a v2 job. Include at least one substantive signal such as issues, note, valuableSources, valuableResults for search, missingContent, querySuggestions, url, or pageNumbers.",
"properties": {
"endpoint": {
"type": "string",
diff --git a/apps/api/src/__tests__/snips/v2/search-feedback.test.ts b/apps/api/src/__tests__/snips/v2/search-feedback.test.ts
index c607837ba..5fa245cfc 100644
--- a/apps/api/src/__tests__/snips/v2/search-feedback.test.ts
+++ b/apps/api/src/__tests__/snips/v2/search-feedback.test.ts
@@ -484,4 +484,163 @@ describeIf(TEST_PRODUCTION)("Search feedback tests", () => {
},
90000,
);
+
+ // Positions are bounded and attributed from the feedback job, which is what
+ // `lookupFeedbackJob` reads. Seeding it directly is how these tests control
+ // the per-source counts without depending on what the live search returned.
+ async function seedFeedbackJob(
+ searchId: string,
+ extra: {
+ numResultsBySource?: Record<string, number>;
+ resultCategories?: Record<string, Record<string, string>>;
+ },
+ ) {
+ let job = await readFeedbackJob(searchId);
+ for (let attempt = 0; attempt < 30 && !job; attempt++) {
+ await new Promise(r => setTimeout(r, 1000));
+ job = await readFeedbackJob(searchId);
+ }
+ expect(job).not.toBeNull();
+ expect(
+ await writeFeedbackJob({
+ endpoint: "search",
+ jobId: searchId,
+ requestId: job!.requestId,
+ teamId: identity.teamId,
+ succeeded: job!.succeeded,
+ creditsBilled: job!.creditsBilled,
+ zeroDataRetention: job!.zeroDataRetention,
+ ...extra,
+ }),
+ ).toBe(true);
+ }
+
+ async function feedbackMetadataFor(feedbackId: string) {
+ const [row] = await db
+ .select({ metadata: schema.search_feedback.metadata })
+ .from(schema.search_feedback)
+ .where(eq(schema.search_feedback.id, feedbackId))
+ .limit(1);
+ expect(row).toBeTruthy();
+ return row.metadata as Record<string, unknown>;
+ }
+
+ it.concurrent(
+ "bounds each group by its own result count",
+ async () => {
+ const raw = await searchRawFull(
+ { query: "firecrawl grouped bounds", limit: 3 },
+ identity,
+ );
+ expect(raw.statusCode).toBe(200);
+ const searchId = raw.body.id;
+
+ await seedFeedbackJob(searchId, {
+ numResultsBySource: { web: 2, images: 0, news: 3 },
+ });
+
+ const result = await searchFeedback(
+ searchId,
+ {
+ rating: "good",
+ valuableResults: [
+ { source: "web", position: 2 },
+ // Past the web group, though the combined total leaves room.
+ { source: "web", position: 3 },
+ { source: "news", position: 3 },
+ // images returned nothing, so no position in it is real.
+ { source: "images", position: 1 },
+ ],
+ },
+ identity,
+ );
+ expect(result.success).toBe(true);
+
+ expect(await feedbackMetadataFor(result.feedbackId)).toEqual(
+ expect.objectContaining({
+ valuableResults: [
+ { source: "web", position: 2 },
+ { source: "news", position: 3 },
+ ],
+ valuableResultDocumentIds: [
+ `search:${searchId}:web:1`,
+ `search:${searchId}:news:2`,
+ ],
+ }),
+ );
+ },
+ 90000,
+ );
+
+ it.concurrent(
+ "records the serving vertical alongside the position",
+ async () => {
+ const raw = await searchRawFull(
+ { query: "firecrawl vertical label", limit: 3 },
+ identity,
+ );
+ expect(raw.statusCode).toBe(200);
+ const searchId = raw.body.id;
+
+ await seedFeedbackJob(searchId, {
+ numResultsBySource: { web: 2, images: 0, news: 0 },
+ resultCategories: { web: { "1": "developer" } },
+ });
+
+ const result = await searchFeedback(
+ searchId,
+ {
+ rating: "good",
+ valuableResults: [
+ { source: "web", position: 1 },
+ // Tagged by nothing, so it carries a position but no category.
+ { source: "web", position: 2 },
+ ],
+ },
+ identity,
+ );
+ expect(result.success).toBe(true);
+
+ expect(await feedbackMetadataFor(result.feedbackId)).toEqual(
+ expect.objectContaining({
+ valuableResults: [
+ { source: "web", position: 1, category: "developer" },
+ { source: "web", position: 2 },
+ ],
+ }),
+ );
+ },
+ 90000,
+ );
+
+ it.concurrent(
+ "keeps positions unbounded when the job predates the counts",
+ async () => {
+ const raw = await searchRawFull(
+ { query: "firecrawl legacy job", limit: 3 },
+ identity,
+ );
+ expect(raw.statusCode).toBe(200);
+ const searchId = raw.body.id;
+
+ // No counts: a job written before the field existed. Unknown bound, so
+ // the label is kept rather than discarded.
+ await seedFeedbackJob(searchId, {});
+
+ const result = await searchFeedback(
+ searchId,
+ { rating: "good", valuableResults: [{ source: "news", position: 2 }] },
+ identity,
+ );
+ expect(result.success).toBe(true);
+
+ expect(await feedbackMetadataFor(result.feedbackId)).toEqual(
+ expect.objectContaining({
+ valuableResults: [{ source: "news", position: 2 }],
+ valuableResultDocumentIds: [`search:${searchId}:news:1`],
+ }),
+ );
+ },
+ 90000,
+ );
});
diff --git a/apps/api/src/controllers/v2/feedback/feedback-store-bounds.test.ts b/apps/api/src/controllers/v2/feedback/feedback-store-bounds.test.ts
new file mode 100644
index 000000000..e57431099
--- /dev/null
+++ b/apps/api/src/controllers/v2/feedback/feedback-store-bounds.test.ts
@@ -0,0 +1,146 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import type { FeedbackJobRow, FeedbackRecordOptions } from "./internal-types";
+
+const mocks = vi.hoisted(() => ({
+ values: vi.fn().mockResolvedValue(undefined),
+}));
+
+vi.mock("../../../db/connection", () => ({
+ db: { insert: () => ({ values: mocks.values }) },
+ dbRr: { select: () => ({}) },
+}));
+
+vi.mock("../../../lib/feedback-job-store", () => ({
+ readFeedbackJob: vi.fn(),
+}));
+
+import { insertFeedback } from "./feedback-store";
+
+const searchId = "01933161-0000-7000-8000-000000000001";
+const teamId = "01933161-0000-7000-8000-000000000002";
+
+function searchJob(overrides: Partial<FeedbackJobRow> = {}): FeedbackJobRow {
+ return {
+ endpoint: "search",
+ id: searchId,
+ request_id: null,
+ team_id: teamId,
+ credits_cost: 2,
+ created_at: new Date().toISOString(),
+ is_successful: true,
+ // The Bigtable feedback job carries no request body.
+ options: null,
+ ...overrides,
+ };
+}
+
+function options(
+ valuableResults: { source: string; position: number }[],
+): FeedbackRecordOptions {
+ return {
+ endpoint: "search",
+ jobId: searchId,
+ feedback: { rating: "good", valuableResults } as any,
+ source: "search_feedback",
+ } as FeedbackRecordOptions;
+}
+
+async function recordedPositions(
+ job: FeedbackJobRow,
+ valuableResults: { source: string; position: number }[],
+) {
+ mocks.values.mockClear();
+ await insertFeedback({
+ feedbackId: "01933161-0000-7000-8000-00000000000f",
+ options: options(valuableResults),
+ job,
+ dbTeamId: teamId,
+ });
+ const row = mocks.values.mock.calls[0]?.[0];
+ return (row?.metadata as { valuableResults?: unknown[] })?.valuableResults;
+}
+
+describe("position bounds from the feedback job's per-source counts", () => {
+ beforeEach(() => vi.clearAllMocks());
+
+ it("bounds each group by its own count", async () => {
+ const job = searchJob({
+ num_results_by_source: { web: 5, images: 0, news: 3 },
+ });
+
+ expect(
+ await recordedPositions(job, [
+ { source: "web", position: 5 },
+ // Past the web group, even though other groups have room.
+ { source: "web", position: 6 },
+ { source: "news", position: 3 },
+ // images returned nothing, so no position in it is real.
+ { source: "images", position: 1 },
+ ]),
+ ).toEqual([
+ { source: "web", position: 5 },
+ { source: "news", position: 3 },
+ ]);
+ });
+
+ it("rejects every position for a group the counts omit", async () => {
+ // A group absent from the counts returned nothing β€” there is no looser
+ // bound to fall back to now that the counts are exact.
+ const job = searchJob({ num_results_by_source: { web: 2 } });
+
+ expect(
+ await recordedPositions(job, [
+ { source: "web", position: 1 },
+ { source: "news", position: 1 },
+ ]),
+ ).toEqual([{ source: "web", position: 1 }]);
+ });
+
+ it("treats an empty count map as a search that returned nothing", async () => {
+ const job = searchJob({ num_results_by_source: {} });
+
+ expect(
+ await recordedPositions(job, [{ source: "web", position: 1 }]),
+ ).toBeUndefined();
+ });
+
+ it("leaves positions unbounded when the job predates the counts", async () => {
+ // Absent is not empty. Feedback is only accepted within
+ // SEARCH_FEEDBACK_MAX_AGE_SEC of the search, so this covers the couple of
+ // minutes spanning the deploy and then closes on its own.
+ const job = searchJob({ num_results_by_source: null });
+
+ expect(
+ await recordedPositions(job, [{ source: "news", position: 2 }]),
+ ).toEqual([{ source: "news", position: 2 }]);
+ });
+
+ it("attributes the serving vertical when the job carries it", async () => {
+ const job = searchJob({
+ num_results_by_source: { web: 3, images: 0, news: 0 },
+ result_categories: { web: { "1": "developer" } },
+ });
+
+ expect(
+ await recordedPositions(job, [
+ { source: "web", position: 1 },
+ { source: "web", position: 2 },
+ ]),
+ ).toEqual([
+ { source: "web", position: 1, category: "developer" },
+ { source: "web", position: 2 },
+ ]);
+ });
+
+ it("drops duplicate and non-positive positions", async () => {
+ const job = searchJob({ num_results_by_source: { web: 3 } });
+
+ expect(
+ await recordedPositions(job, [
+ { source: "web", position: 1 },
+ { source: "web", position: 1 },
+ { source: "web", position: 0 },
+ ]),
+ ).toEqual([{ source: "web", position: 1 }]);
+ });
+});
diff --git a/apps/api/src/controllers/v2/feedback/feedback-store.ts b/apps/api/src/controllers/v2/feedback/feedback-store.ts
index d0f84580e..c985c3e2a 100644
--- a/apps/api/src/controllers/v2/feedback/feedback-store.ts
+++ b/apps/api/src/controllers/v2/feedback/feedback-store.ts
@@ -8,10 +8,12 @@ import {
} from "../../../lib/feedback-job-store";
import { logger } from "../../../lib/logger";
import { EndpointFeedbackEndpoint } from "../types";
+import type { SearchResultType } from "../../../lib/entities";
import {
FeedbackJobRow,
FeedbackRecordOptions,
RefundPolicySnapshot,
+ ValuableResultInput,
} from "./internal-types";
type DbError = { code?: string } & Record<string, unknown>;
@@ -21,8 +23,192 @@ type ExistingFeedback = {
credits_refunded: number | null;
};
+type ValuableResultDocument = {
+ documentId: string;
+ searchId: string;
+ /** The `data` group the client addressed, as sent: web | images | news. */
+ requestedSource: SearchResultType;
+ /** Analytics spelling, as used in `documentId`. */
+ resultType: SearchResultDocumentType;
+ resultIndex: number;
+ position: number;
+ /**
+ * Which vertical served this result, as the response tagged it β€” `developer`
+ * for an index-served hit, `github` / `research` / `pdf` for a URL-derived
+ * one. Absent when the result carried no category, or when the feedback job
+ * predates `resultCategories`. This is the attribution half of the label:
+ * `requestedSource` says where in `data` the result sat, `category` says who
+ * answered.
+ */
+ category?: string;
+ reason?: string;
+ source: "position";
+};
+
+/**
+ * Result type as recorded in analytics. The API groups image results under
+ * `images` (matching `data.images`), but the ClickHouse `search_results` table
+ * writes the singular `image`; document IDs use the analytics spelling so they
+ * join directly on (search_id, result_type, result_index).
+ */
+type SearchResultDocumentType = "web" | "news" | "image";
+
+const RESULT_DOCUMENT_TYPES: Record<
+ SearchResultType,
+ SearchResultDocumentType
+> = {
+ web: "web",
+ news: "news",
+ images: "image",
+};
+
+function searchFeedbackResultDocumentId(
+ searchId: string,
+ resultType: SearchResultDocumentType,
+ resultIndex: number,
+): string {
+ return `search:${searchId}:${resultType}:${resultIndex}`;
+}
+
+/**
+ * Number of results each `data` group returned, as the feedback job recorded
+ * it. Returns null when the job predates the field or carries an unrecognised
+ * shape β€” which is not the same as an empty object, meaning the search ran and
+ * every group was empty.
+ */
+function resultCountsBySource(
+ job: FeedbackJobRow,
+): Partial<Record<SearchResultType, number>> | null {
+ const counts = job.num_results_by_source;
+ if (counts === null || counts === undefined || typeof counts !== "object") {
+ return null;
+ }
+
+ // An object with no usable entries still counts as known: the search ran
+ // and no group this API can address returned anything, which rejects every
+ // position. Only a missing or non-object value is unknown.
+ const entries = Object.entries(counts as Record<string, unknown>).filter(
+ (entry): entry is [SearchResultType, number] =>
+ entry[0] in RESULT_DOCUMENT_TYPES &&
+ typeof entry[1] === "number" &&
+ Number.isInteger(entry[1]) &&
+ entry[1] >= 0,
+ );
+ return Object.fromEntries(entries);
+}
+
+/**
+ * Vertical that served each position, as the feedback job recorded it. Returns
+ * null when the job predates the field or carries an unrecognised shape, which
+ * again is not the same as an empty object.
+ */
+function resultCategoriesBySource(
+ job: FeedbackJobRow,
+): Partial<Record<SearchResultType, Record<string, string>>> | null {
+ const categories = job.result_categories;
+ if (
+ categories === null ||
+ categories === undefined ||
+ typeof categories !== "object"
+ ) {
+ return null;
+ }
+
+ const bySource: Partial<Record<SearchResultType, Record<string, string>>> =
+ {};
+ for (const [source, positions] of Object.entries(
+ categories as Record<string, unknown>,
+ )) {
+ if (!(source in RESULT_DOCUMENT_TYPES)) continue;
+ if (!positions || typeof positions !== "object") continue;
+
+ const byPosition: Record<string, string> = {};
+ for (const [position, category] of Object.entries(
+ positions as Record<string, unknown>,
+ )) {
+ if (typeof category === "string" && category.length > 0) {
+ byPosition[position] = category;
+ }
+ }
+ bySource[source as SearchResultType] = byPosition;
+ }
+
+ return bySource;
+}
+
+/**
+ * Highest 1-indexed position addressable in `data[source]`, or null when the
+ * job carries no usable bound.
+ *
+ * The per-source counts bound every group exactly, so there is no looser
+ * fallback to reach for: a group the counts do not mention returned nothing
+ * and has no valid position. Null means the job predates the field β€” feedback
+ * is only accepted within SEARCH_FEEDBACK_MAX_AGE_SEC of the search, so that
+ * only covers searches run in the couple of minutes spanning the deploy, and
+ * closes on its own.
+ */
+function maxResultPosition(
+ job: FeedbackJobRow,
+ source: SearchResultType,
+): number | null {
+ const counts = resultCountsBySource(job);
+ if (counts === null) return null;
+ return counts[source] ?? 0;
+}
+
+function valuableResultDocuments(
+ job: FeedbackJobRow,
+ results: ValuableResultInput[] | undefined,
+): ValuableResultDocument[] {
+ if (job.endpoint !== "search" || !results?.length) return [];
+
+ // Rejects hallucinated positions, which would otherwise be stored as
+ // false-positive relevance labels. Zero is a real bound: a group that
+ // returned nothing has no valuable positions.
+ const maxPositions = new Map<SearchResultType, number | null>();
+ const seen = new Set<string>();
+ const categories = resultCategoriesBySource(job);
+
+ return results.flatMap(({ source, position, reason }) => {
+ if (!(source in RESULT_DOCUMENT_TYPES)) return [];
+ if (!Number.isInteger(position) || position <= 0) return [];
+
+ if (!maxPositions.has(source)) {
+ maxPositions.set(source, maxResultPosition(job, source));
+ }
+ const maxPosition = maxPositions.get(source) ?? null;
+ if (maxPosition !== null && position > maxPosition) return [];
+
+ const key = `${source}:${position}`;
+ if (seen.has(key)) return [];
+ seen.add(key);
+
+ const resultType = RESULT_DOCUMENT_TYPES[source];
+ const resultIndex = position - 1;
+ const category = categories?.[source]?.[String(position)];
+ return [
+ {
+ documentId: searchFeedbackResultDocumentId(
+ job.id,
+ resultType,
+ resultIndex,
+ ),
+ searchId: job.id,
+ requestedSource: source,
+ resultType,
+ resultIndex,
+ position,
+ ...(category ? { category } : {}),
+ ...(reason ? { reason } : {}),
+ source: "position" as const,
+ },
+ ];
+ });
+}
+
function feedbackMetadata(
options: FeedbackRecordOptions,
+ valuableResultDocs: ValuableResultDocument[],
): Record<string, unknown> {
return {
...(options.feedback.metadata ?? {}),
@@ -30,6 +216,20 @@ function feedbackMetadata(
...(options.feedback.pageNumbers
? { pageNumbers: options.feedback.pageNumbers }
: {}),
+ ...(valuableResultDocs.length > 0
+ ? {
+ // Echoed back in the vocabulary the client sent, not the analytics
+ // spelling used in the document IDs.
+ valuableResults: valuableResultDocs.map(doc => ({
+ source: doc.requestedSource,
+ position: doc.position,
+ ...(doc.category ? { category: doc.category } : {}),
+ })),
+ valuableResultDocumentIds: valuableResultDocs.map(
+ doc => doc.documentId,
+ ),
+ }
+ : {}),
};
}
@@ -72,6 +272,10 @@ export async function lookupFeedbackJob(
feedback_deadline_ms: job.feedbackDeadlineMs,
refund_class: job.refundClass,
zero_data_retention: job.zeroDataRetention,
+ num_results_by_source:
+ endpoint === "search" ? (job.numResultsBySource ?? null) : null,
+ result_categories:
+ endpoint === "search" ? (job.resultCategories ?? null) : null,
};
}
@@ -96,6 +300,10 @@ export async function insertFeedback(params: {
apiKeyId?: number | null;
}): Promise<DbError | null> {
const { feedbackId, options, job, dbTeamId, apiKeyId } = params;
+ const valuableResultDocs = valuableResultDocuments(
+ job,
+ options.feedback.valuableResults,
+ );
try {
await db.insert(schema.search_feedback).values({
id: feedbackId,
@@ -110,10 +318,13 @@ export async function insertFeedback(params: {
issue_types: options.feedback.issues ?? [],
tags: options.feedback.tags ?? [],
comment: options.feedback.note ?? null,
- valuable_sources: options.feedback.valuableSources ?? [],
+ valuable_sources: [
+ ...(options.feedback.valuableSources ?? []),
+ ...valuableResultDocs,
+ ],
missing_content: options.feedback.missingContent ?? [],
query_suggestions: options.feedback.querySuggestions ?? null,
- metadata: feedbackMetadata(options),
+ metadata: feedbackMetadata(options, valuableResultDocs),
job_status: job.is_successful === false ? "failed" : "completed",
credits_billed: job.credits_cost ?? 0,
credits_refunded: 0,
diff --git a/apps/api/src/controllers/v2/feedback/internal-types.ts b/apps/api/src/controllers/v2/feedback/internal-types.ts
index b9b196c4c..8cfc509c9 100644
--- a/apps/api/src/controllers/v2/feedback/internal-types.ts
+++ b/apps/api/src/controllers/v2/feedback/internal-types.ts
@@ -1,4 +1,5 @@
import { logger as _logger } from "../../../lib/logger";
+import type { SearchResultType } from "../../../lib/entities";
import {
EndpointFeedbackEndpoint,
EndpointFeedbackErrorCode,
@@ -9,12 +10,20 @@ import type { RefundClass } from "../../../lib/feedback-job-store";
export type FeedbackRating = "good" | "partial" | "bad";
+/** A valuable result addressed by its group and 1-indexed position in it. */
+export type ValuableResultInput = {
+ source: SearchResultType;
+ position: number;
+ reason?: string;
+};
+
export type FeedbackInput = {
rating: FeedbackRating;
issues?: string[];
tags?: string[];
note?: string;
valuableSources?: Array<{ url: string; reason?: string }>;
+ valuableResults?: ValuableResultInput[];
missingContent?: Array<{ topic: string; description?: string }>;
querySuggestions?: string;
url?: string;
@@ -33,6 +42,19 @@ export type FeedbackJobRow = {
created_at: string;
is_successful: boolean | null;
options: unknown;
+ /**
+ * Per-source result counts (search only), e.g. `{"web":3,"images":0}`, read
+ * from the Bigtable feedback job. Null on jobs written before the field
+ * existed; `{}` means the search ran and every group was empty.
+ */
+ num_results_by_source?: unknown;
+ /**
+ * Which vertical served each position (search only), e.g.
+ * `{"web":{"1":"developer"}}`. Sparse: untagged results and untagged groups
+ * are absent. Null on jobs written before the field existed and on
+ * zero-data-retention searches.
+ */
+ result_categories?: unknown;
feedback_deadline_ms?: number;
refund_class?: RefundClass;
zero_data_retention?: boolean;
diff --git a/apps/api/src/controllers/v2/feedback/record-options.test.ts b/apps/api/src/controllers/v2/feedback/record-options.test.ts
index 477dc5c52..d458f9b28 100644
--- a/apps/api/src/controllers/v2/feedback/record-options.test.ts
+++ b/apps/api/src/controllers/v2/feedback/record-options.test.ts
@@ -97,6 +97,49 @@ describe("feedback schema", () => {
}).success,
).toBe(true);
+ expect(
+ searchFeedbackSchema.safeParse({
+ rating: "good",
+ valuableResults: [
+ { source: "web", position: 1 },
+ { source: "news", position: 2, reason: "Broke the story." },
+ { source: "images", position: 1 },
+ ],
+ }).success,
+ ).toBe(true);
+
+ expect(
+ endpointFeedbackSchema.safeParse({
+ endpoint: "search",
+ jobId: "01933161-0000-7000-8000-000000000001",
+ rating: "good",
+ valuableResults: [{ source: "web", position: 1 }],
+ }).success,
+ ).toBe(true);
+
+ // `source` is required β€” a bare position is ambiguous across groups.
+ expect(
+ searchFeedbackSchema.safeParse({
+ rating: "good",
+ valuableResults: [{ position: 1 }],
+ }).success,
+ ).toBe(false);
+
+ expect(
+ searchFeedbackSchema.safeParse({
+ rating: "good",
+ valuableResults: [{ source: "video", position: 1 }],
+ }).success,
+ ).toBe(false);
+
+ // The old flat form is no longer accepted.
+ expect(
+ searchFeedbackSchema.safeParse({
+ rating: "good",
+ valuableResultPositions: [1, 2],
+ }).success,
+ ).toBe(false);
+
expect(
endpointFeedbackSchema.safeParse({
endpoint: "search",
@@ -114,5 +157,35 @@ describe("feedback schema", () => {
note: "Generic endpoint feedback remains valid with a note.",
}).success,
).toBe(true);
+
+ expect(
+ endpointFeedbackSchema.safeParse({
+ endpoint: "map",
+ jobId: "01933161-0000-7000-8000-000000000001",
+ rating: "good",
+ valuableResults: [{ source: "web", position: 1 }],
+ }).success,
+ ).toBe(false);
+
+ expect(
+ searchFeedbackSchema.safeParse({
+ rating: "good",
+ valuableResults: [{ source: "web", position: 0 }],
+ }).success,
+ ).toBe(false);
+
+ expect(
+ searchFeedbackSchema.safeParse({
+ rating: "good",
+ valuableResults: [{ source: "web", position: 101 }],
+ }).success,
+ ).toBe(false);
+
+ expect(
+ searchFeedbackSchema.safeParse({
+ rating: "good",
+ valuableResults: [{ source: "web", position: 100 }],
+ }).success,
+ ).toBe(true);
});
});
diff --git a/apps/api/src/controllers/v2/feedback/request-input.ts b/apps/api/src/controllers/v2/feedback/request-input.ts
index d05d2d09a..3e3370452 100644
--- a/apps/api/src/controllers/v2/feedback/request-input.ts
+++ b/apps/api/src/controllers/v2/feedback/request-input.ts
@@ -7,6 +7,7 @@ export function toFeedbackInput(
return {
rating: body.rating,
valuableSources: body.valuableSources,
+ valuableResults: body.valuableResults,
missingContent: body.missingContent,
querySuggestions: body.querySuggestions,
origin: body.origin,
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index 568bdf28d..141bc9fb7 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -539,6 +539,8 @@ async function searchControllerInner(
error: undefined,
results: result.response as any,
num_results: result.totalResultsCount,
+ num_results_by_source: result.resultCountsBySource,
+ result_categories: result.resultCategories,
time_taken: timeTakenInSeconds,
team_id: req.auth.team_id,
options: req.body,
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 2c9e875e0..c051c9b9a 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -2845,22 +2845,40 @@ const missingContentEntrySchema = z.strictObject({
description: z.string().trim().max(2000).optional(),
});
+/**
+ * A single valuable search result, addressed by the group it appears in and its
+ * 1-indexed position within that group. Results are returned grouped
+ * (`data.web`, `data.images`, `data.news`), each group sliced independently to
+ * `limit`, so a bare position is ambiguous once more than one source is in play.
+ */
+const valuableResultSchema = z.strictObject({
+ source: z.enum(["web", "images", "news"]),
+ // 100 is the max search `limit`, and each group is capped at `limit`
+ // independently, so positions beyond it can only be hallucinated.
+ position: z.number().int().positive().lte(100),
+ reason: z.string().trim().max(1000).optional(),
+});
+
+const valuableResultsSchema = z.array(valuableResultSchema).max(50);
+
function hasSubstantiveSearchFeedback(data: {
rating: "good" | "bad" | "partial";
valuableSources?: unknown[];
+ valuableResults?: unknown[];
missingContent?: unknown[];
querySuggestions?: string;
}): boolean {
const hasSources = (data.valuableSources?.length ?? 0) > 0;
+ const hasResults = (data.valuableResults?.length ?? 0) > 0;
const hasMissing = (data.missingContent?.length ?? 0) > 0;
const hasSuggestions =
!!data.querySuggestions && data.querySuggestions.length > 0;
switch (data.rating) {
case "good":
- return hasSources;
+ return hasSources || hasResults;
case "partial":
- return hasSources || hasMissing;
+ return hasSources || hasResults || hasMissing;
case "bad":
return hasMissing || hasSuggestions;
}
@@ -2880,6 +2898,7 @@ export const searchFeedbackSchema = z
)
.max(50)
.optional(),
+ valuableResults: valuableResultsSchema.optional(),
missingContent: z.array(missingContentEntrySchema).max(20).optional(),
querySuggestions: z.string().trim().max(2000).optional(),
origin: z.string().optional().prefault("api"),
@@ -2887,7 +2906,7 @@ export const searchFeedbackSchema = z
})
.refine(data => hasSubstantiveSearchFeedback(data), {
message:
- "Feedback must be substantive. 'good' requires at least one valuableSources entry; 'partial' requires valuableSources or at least one missingContent entry; 'bad' requires at least one missingContent entry or querySuggestions.",
+ "Feedback must be substantive. 'good' requires valuableSources or valuableResults; 'partial' requires valuableSources, valuableResults, or at least one missingContent entry; 'bad' requires at least one missingContent entry or querySuggestions.",
});
export type SearchFeedbackRequest = z.infer<typeof searchFeedbackSchema>;
@@ -2968,6 +2987,7 @@ export const endpointFeedbackSchema = z
)
.max(50)
.optional(),
+ valuableResults: valuableResultsSchema.optional(),
missingContent: z.array(missingContentEntrySchema).max(50).optional(),
querySuggestions: z.string().trim().max(2000).optional(),
url: searchFeedbackUrlSchema.optional(),
@@ -2983,6 +3003,7 @@ export const endpointFeedbackSchema = z
(data.tags?.length ?? 0) > 0 ||
(data.note?.length ?? 0) > 0 ||
(data.valuableSources?.length ?? 0) > 0 ||
+ (data.valuableResults?.length ?? 0) > 0 ||
(data.missingContent?.length ?? 0) > 0 ||
!!data.querySuggestions ||
!!data.url ||
@@ -2991,14 +3012,21 @@ export const endpointFeedbackSchema = z
},
{
message:
- "Feedback must include at least one substantive signal: issues, note, sources, missingContent, querySuggestions, url, or pageNumbers.",
+ "Feedback must include at least one substantive signal: issues, note, sources, valuableResults, missingContent, querySuggestions, url, or pageNumbers.",
+ },
+ )
+ .refine(
+ data =>
+ data.endpoint === "search" || (data.valuableResults?.length ?? 0) === 0,
+ {
+ message: "valuableResults is only supported for search feedback.",
},
)
.refine(
data => data.endpoint !== "search" || hasSubstantiveSearchFeedback(data),
{
message:
- "Search feedback must be substantive. 'good' requires at least one valuableSources entry; 'partial' requires valuableSources or at least one missingContent entry; 'bad' requires at least one missingContent entry or querySuggestions.",
+ "Search feedback must be substantive. 'good' requires valuableSources or valuableResults; 'partial' requires valuableSources, valuableResults, or at least one missingContent entry; 'bad' requires at least one missingContent entry or querySuggestions.",
},
);
diff --git a/apps/api/src/lib/__tests__/search-result-categories.test.ts b/apps/api/src/lib/__tests__/search-result-categories.test.ts
new file mode 100644
index 000000000..56ef94303
--- /dev/null
+++ b/apps/api/src/lib/__tests__/search-result-categories.test.ts
@@ -0,0 +1,115 @@
+import { describe, it, expect } from "vitest";
+import {
+ collectSearchResultCategories,
+ countSearchResultsBySource,
+ type SearchV2Response,
+} from "../entities";
+
+const web = (url: string, category?: string) => ({
+ url,
+ title: url,
+ description: "",
+ ...(category ? { category } : {}),
+});
+
+describe("collectSearchResultCategories", () => {
+ it("keys the serving vertical by group and 1-indexed position", () => {
+ const response: SearchV2Response = {
+ web: [
+ web("https://github.com/firecrawl/firecrawl", "github"),
+ web("https://example.com/"),
+ web("https://arxiv.org/abs/1", "research"),
+ ],
+ };
+
+ expect(collectSearchResultCategories(response)).toEqual({
+ web: { "1": "github", "3": "research" },
+ });
+ });
+
+ it("tags every developer hit, which is the whole web group", () => {
+ // The developer category is exclusive, so its hits ARE data.web β€”
+ // renumbered from 1 after threat filtering.
+ const response: SearchV2Response = {
+ web: [
+ web("https://github.com/firecrawl/firecrawl/issues/1", "developer"),
+ web("https://docs.firecrawl.dev/", "developer"),
+ ],
+ };
+
+ expect(collectSearchResultCategories(response)).toEqual({
+ web: { "1": "developer", "2": "developer" },
+ });
+ });
+
+ it("distinguishes an untagged search from an absent one", () => {
+ // `{}` is a claim: the search ran and nothing carried a category. NULL in
+ // the column is the separate claim that the row cannot answer at all.
+ expect(
+ collectSearchResultCategories({ web: [web("https://example.com/")] }),
+ ).toEqual({});
+ expect(collectSearchResultCategories({})).toEqual({});
+ });
+
+ it("omits groups with no tagged results rather than emitting empty maps", () => {
+ const response: SearchV2Response = {
+ web: [web("https://github.com/firecrawl/cli", "github")],
+ news: [{ url: "https://news.example/a", title: "A" }],
+ images: [{ url: "https://img.example/a.png", title: "A" }],
+ };
+
+ expect(collectSearchResultCategories(response)).toEqual({
+ web: { "1": "github" },
+ });
+ });
+
+ it("keys news positions independently of web", () => {
+ const response: SearchV2Response = {
+ web: [web("https://example.com/")],
+ news: [
+ { url: "https://news.example/a", title: "A", category: "research" },
+ ],
+ };
+
+ expect(collectSearchResultCategories(response)).toEqual({
+ news: { "1": "research" },
+ });
+ });
+
+ it("ignores non-string and empty categories", () => {
+ const response = {
+ web: [
+ { url: "https://a.example/", title: "", description: "", category: "" },
+ {
+ url: "https://b.example/",
+ title: "",
+ description: "",
+ category: 7 as unknown as string,
+ },
+ web("https://c.example/", "pdf"),
+ ],
+ } as SearchV2Response;
+
+ expect(collectSearchResultCategories(response)).toEqual({
+ web: { "3": "pdf" },
+ });
+ });
+
+ it("positions line up with the counts feedback bounds against", () => {
+ const response: SearchV2Response = {
+ web: [web("https://a.example/", "developer"), web("https://b.example/")],
+ news: [{ url: "https://news.example/a", title: "A" }],
+ };
+
+ const counts = countSearchResultsBySource(response);
+ const categories = collectSearchResultCategories(response);
+
+ for (const [source, byPosition] of Object.entries(categories)) {
+ for (const position of Object.keys(byPosition)) {
+ expect(Number(position)).toBeLessThanOrEqual(
+ counts[source as keyof typeof counts],
+ );
+ }
+ }
+ });
+});
diff --git a/apps/api/src/lib/entities.ts b/apps/api/src/lib/entities.ts
index 440bf134f..179f79f30 100644
--- a/apps/api/src/lib/entities.ts
+++ b/apps/api/src/lib/entities.ts
@@ -170,6 +170,73 @@ export interface SearchV2Response {
news?: NewsSearchResult[];
}
+/**
+ * Length of each `SearchV2Response` group as returned to the client. Every
+ * group is present and non-negative β€” a source that was not requested, or that
+ * returned nothing, counts 0. Persisted per search so position-based feedback
+ * can be bounded exactly per source rather than against a combined total.
+ */
+export type SearchResultCountsBySource = Record<SearchResultType, number>;
+
+export function countSearchResultsBySource(
+ response: SearchV2Response,
+): SearchResultCountsBySource {
+ return {
+ web: response.web?.length ?? 0,
+ images: response.images?.length ?? 0,
+ news: response.news?.length ?? 0,
+ };
+}
+
+/**
+ * Which vertical produced each result, keyed by group and then by the result's
+ * 1-indexed position in that group: `{"web":{"1":"developer","3":"github"}}`.
+ *
+ * Sparse on purpose. Results carry a `category` only when one applies β€” the
+ * developer index stamps `developer` on every hit it serves (see
+ * `search/developer.ts`), and `github` / `research` / `pdf` are derived per URL
+ * in `search/execute.ts` β€” so untagged results are simply absent, as are groups
+ * with no tagged results at all. An empty object therefore means "nothing was
+ * tagged", which is a different claim from a NULL column ("this row predates
+ * the column").
+ *
+ * This is the only place the serving vertical survives the response. `source`
+ * says which `data` group a result sits in, which is addressing; `category`
+ * says who answered, which is attribution. They are the same value today only
+ * because the developer category is schema-enforced as exclusive and lands in
+ * `data.web`; blend a vertical into ordinary web results and the two come
+ * apart. Persisted per search so position-based feedback can resolve the
+ * vertical behind a `(source, position)` pair it is handed later.
+ */
+export type SearchResultCategoriesBySource = Partial<
+ Record<SearchResultType, Record<string, string>>
+>;
+
+export function collectSearchResultCategories(
+ response: SearchV2Response,
+): SearchResultCategoriesBySource {
+ const collected: SearchResultCategoriesBySource = {};
+
+ for (const source of ["web", "images", "news"] as const) {
+ const results = response[source];
+ if (!results?.length) continue;
+
+ const categories: Record<string, string> = {};
+ results.forEach((result, index) => {
+ const category = (result as { category?: unknown }).category;
+ if (typeof category === "string" && category.length > 0) {
+ categories[String(index + 1)] = category;
+ }
+ });
+
+ if (Object.keys(categories).length > 0) {
+ collected[source] = categories;
+ }
+ }
+
+ return collected;
+}
+
export interface ScrapeActionContent {
url: string;
html: string;
diff --git a/apps/api/src/lib/feedback-job-store.ts b/apps/api/src/lib/feedback-job-store.ts
index 06b542b09..448141d0d 100644
--- a/apps/api/src/lib/feedback-job-store.ts
+++ b/apps/api/src/lib/feedback-job-store.ts
@@ -39,6 +39,23 @@ type FeedbackJob = {
succeeded: boolean;
creditsBilled: number;
zeroDataRetention: boolean;
+ /**
+ * Search only. How many results each group returned, e.g.
+ * `{"web":3,"images":0,"news":10}`, and which vertical served each position,
+ * e.g. `{"web":{"1":"developer"}}`.
+ *
+ * Position feedback addresses a result as `{source, position}`, 1-indexed
+ * within its group, so bounding a position needs the length of that group
+ * and nothing else. They live here rather than beside the search row
+ * because this is the record the feedback window is enforced from, and it
+ * is the only one feedback reads.
+ *
+ * Absent on records written before these fields existed, on non-search
+ * jobs, and under zero data retention. Absent is not empty: `{}` means the
+ * search ran and every group was empty, which rejects every position.
+ */
+ numResultsBySource?: Record<string, number>;
+ resultCategories?: Record<string, Record<string, string>>;
};
function parseFeedbackJob(value: Buffer | string): FeedbackJob {
@@ -132,6 +149,8 @@ type FeedbackJobBase = {
creditsBilled: number;
zeroDataRetention: boolean;
completedAt?: Date;
+ numResultsBySource?: Record<string, number>;
+ resultCategories?: Record<string, Record<string, string>>;
};
type FeedbackJobWrite = FeedbackJobBase &
@@ -187,6 +206,19 @@ export async function writeFeedbackJob(
succeeded: params.succeeded,
creditsBilled: params.creditsBilled,
zeroDataRetention: params.zeroDataRetention,
+ // Both describe the response, so zero data retention redacts them
+ // with everything else. Feedback on a ZDR job is dropped before the
+ // bounds are read, so nothing needs them for those rows.
+ ...(params.zeroDataRetention
+ ? {}
+ : {
+ ...(params.numResultsBySource
+ ? { numResultsBySource: params.numResultsBySource }
+ : {}),
+ ...(params.resultCategories
+ ? { resultCategories: params.resultCategories }
+ : {}),
+ }),
}),
);
const table = await getBigtableTable(tableId);
diff --git a/apps/api/src/search/execute.test.ts b/apps/api/src/search/execute.test.ts
index 9090c21bb..3a1e558c2 100644
--- a/apps/api/src/search/execute.test.ts
+++ b/apps/api/src/search/execute.test.ts
@@ -219,6 +219,9 @@ describe("executeSearch developer category", () => {
expect(result.response).toEqual({ web: [developerResult] });
expect(result.response).not.toHaveProperty("developer");
expect(result.indexResultsCount).toBe(1);
+ // The vertical that served the result survives the response as a label
+ // keyed by the position the client sees.
+ expect(result.resultCategories).toEqual({ web: { "1": "developer" } });
});
it("filters blocked developer results via threat protection and renumbers", async () => {
@@ -263,6 +266,47 @@ describe("executeSearch developer category", () => {
["https://ok.example/a", 1],
["https://ok.example/c", 2],
]);
+ // Keyed by the renumbered positions, not the upstream ones β€” position 2 is
+ // the surviving third result.
+ expect(result.resultCategories).toEqual({
+ web: { "1": "developer", "2": "developer" },
+ });
+ });
+
+ it("returns an empty group and no categories when every result is blocked", async () => {
+ mocks.searchDeveloperCategory.mockResolvedValue([
+ {
+ url: "https://blocked.example/a",
+ title: "A",
+ description: "",
+ position: 1,
+ category: "developer",
+ },
+ {
+ url: "https://blocked.example/b",
+ title: "B",
+ description: "",
+ position: 2,
+ category: "developer",
+ },
+ ]);
+ mocks.checkUrlsAgainstThreatPolicy.mockResolvedValue({
+ decisionsByUrl: new Map([
+ ["https://blocked.example/a", { allowed: false }],
+ ["https://blocked.example/b", { allowed: false }],
+ ]),
+ });
+
+ const result = await executeSearch(
+ options([{ type: "developer" }]),
+ { ...context, threatProtectionPolicy: { mode: "block" } } as any,
+ logger,
+ );
+
+ expect(result.response.web ?? []).toEqual([]);
+ // An empty map, not absent: the search ran and tagged nothing, which
+ // feedback must tell apart from a row that predates the column.
+ expect(result.resultCategories).toEqual({});
});
it("rejects developer combined with other categories at the schema", () => {
diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.ts
index 257fcace3..903532aad 100644
--- a/apps/api/src/search/execute.ts
+++ b/apps/api/src/search/execute.ts
@@ -1,7 +1,14 @@
import type { Logger } from "winston";
import { discoverTools, isAlexandriaSource } from "./alexandria";
import { search } from "./v2";
-import { SearchV2Response, SearchResultType } from "../lib/entities";
+import {
+ SearchV2Response,
+ SearchResultType,
+ SearchResultCountsBySource,
+ SearchResultCategoriesBySource,
+ countSearchResultsBySource,
+ collectSearchResultCategories,
+} from "../lib/entities";
import {
buildSearchQuery,
getCategoryFromUrl,
@@ -77,6 +84,8 @@ interface SearchExecuteResult {
toolsUnavailable?: boolean;
response: SearchV2Response;
totalResultsCount: number;
+ resultCountsBySource: SearchResultCountsBySource;
+ resultCategories: SearchResultCategoriesBySource;
indexResultsCount: number;
searchCredits: number;
scrapeCredits: number;
@@ -453,6 +462,13 @@ export async function executeSearch(
toolsWarning,
toolsUnavailable,
totalResultsCount,
+ // Counted from the final response β€” after scraping and highlights β€” so it
+ // matches exactly what the client can address by position.
+ resultCountsBySource: countSearchResultsBySource(searchResponse),
+ // Read from the same final response, so a category is keyed by the position
+ // the client sees. Index hits replace the web group above and are
+ // renumbered there, so their positions here are the renumbered ones.
+ resultCategories: collectSearchResultCategories(searchResponse),
indexResultsCount,
searchCredits,
scrapeCredits,
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index 6cfabebe6..62243c3c0 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -16,6 +16,10 @@ import {
import { hasFormatOfType } from "../../lib/format-utils";
import { keylessTeamUuid } from "../../lib/keyless";
import type { Document, ScrapeOptions } from "../../controllers/v2/types";
+import type {
+ SearchResultCountsBySource,
+ SearchResultCategoriesBySource,
+} from "../../lib/entities";
import type { CostTracking } from "../../lib/cost-tracking";
import type { Logger } from "winston";
import { saveExtractResult } from "../../lib/extract/extract-redis";
@@ -983,6 +987,8 @@ export type LoggedSearch = {
is_successful: boolean;
error?: string;
num_results: number;
+ num_results_by_source?: SearchResultCountsBySource;
+ result_categories?: SearchResultCategoriesBySource;
results: any;
zeroDataRetention: boolean;
};
@@ -1028,6 +1034,18 @@ async function logSearchInternal(search: LoggedSearch) {
succeeded: search.is_successful,
creditsBilled: search.credits_cost,
zeroDataRetention: search.zeroDataRetention,
+ // Position feedback bounds each `{source, position}` against the group
+ // it addresses, and attributes it to the vertical that served it. This
+ // is the record feedback reads, and the one the feedback window is
+ // enforced from, so the counts travel with it. `writeFeedbackJob`
+ // redacts both under zero data retention.
+ //
+ // Only v2 search reports counts. A v0/v1 search returns a flat list
+ // whose positions do not map to `{source, position}`, so it records `{}`
+ // and every position is rejected. Leaving it absent would read as a job
+ // that predates the field and accept any position unbounded.
+ numResultsBySource: search.num_results_by_source ?? {},
+ resultCategories: search.result_categories,
},
logger,
);
@@ -1048,6 +1066,18 @@ async function logSearchInternal(search: LoggedSearch) {
is_successful: search.is_successful,
error: search.zeroDataRetention ? null : (search.error ?? null),
num_results: search.num_results,
+ // Both are response-derived and both are redacted under zero data
+ // retention: the per-source split says how the result set was composed
+ // ({"web":3,"news":10} characterises the query's coverage in a way the
+ // combined total does not), and the category map says which vertical
+ // served each position. Feedback on a ZDR search is not persisted
+ // anyway, so neither column has a reader for those rows.
+ num_results_by_source: search.zeroDataRetention
+ ? null
+ : (search.num_results_by_source ?? null),
+ result_categories: search.zeroDataRetention
+ ? null
+ : (search.result_categories ?? null),
time_taken: search.time_taken,
},
logger,