Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): add generic endpoint feedback (#3693)
Abimael Martell92243e807821
diff --git a/apps/api/openapi.json b/apps/api/openapi.jsonindex 101a62aeb..249678f56 100644--- a/apps/api/openapi.json+++ b/apps/api/openapi.json@@ -1911,6 +1911,186 @@}}},+ "/feedback": {+ "post": {+ "summary": "Submit feedback for a v2 job",+ "operationId": "submitEndpointFeedback",+ "tags": ["Feedback"],+ "security": [+ {+ "bearerAuth": []+ }+ ],+ "requestBody": {+ "required": true,+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/EndpointFeedbackRequest"+ }+ }+ }+ },+ "responses": {+ "200": {+ "description": "Feedback recorded",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/FeedbackResponse"+ }+ }+ }+ },+ "400": {+ "description": "Invalid request body",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/FeedbackErrorResponse"+ }+ }+ }+ },+ "403": {+ "description": "Feedback is not available for this team",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/FeedbackErrorResponse"+ }+ }+ }+ },+ "404": {+ "description": "Job not found for this team",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/FeedbackErrorResponse"+ }+ }+ }+ },+ "409": {+ "description": "Feedback cannot be recorded for this job",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/FeedbackErrorResponse"+ }+ }+ }+ },+ "500": {+ "description": "Server error",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/FeedbackErrorResponse"+ }+ }+ }+ }+ }+ }+ },+ "/search/{jobId}/feedback": {+ "post": {+ "summary": "Submit feedback for a search job",+ "operationId": "submitSearchFeedback",+ "tags": ["Search", "Feedback"],+ "security": [+ {+ "bearerAuth": []+ }+ ],+ "parameters": [+ {+ "name": "jobId",+ "in": "path",+ "required": true,+ "schema": {+ "type": "string",+ "format": "uuid"+ },+ "description": "Search job id returned by /search."+ }+ ],+ "requestBody": {+ "required": true,+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/SearchFeedbackRequest"+ }+ }+ }+ },+ "responses": {+ "200": {+ "description": "Feedback recorded",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/FeedbackResponse"+ }+ }+ }+ },+ "400": {+ "description": "Invalid request body",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/FeedbackErrorResponse"+ }+ }+ }+ },+ "403": {+ "description": "Feedback is not available for this team",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/FeedbackErrorResponse"+ }+ }+ }+ },+ "404": {+ "description": "Search not found for this team",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/FeedbackErrorResponse"+ }+ }+ }+ },+ "409": {+ "description": "Feedback cannot be recorded for this search",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/FeedbackErrorResponse"+ }+ }+ }+ },+ "500": {+ "description": "Server error",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/FeedbackErrorResponse"+ }+ }+ }+ }+ }+ }+ },"/llmstxt": {"post": {"summary": "Generate LLMs.txt for a website",@@ -2919,6 +3099,11 @@"success": {"type": "boolean"},+ "id": {+ "type": "string",+ "format": "uuid",+ "description": "The map job id. Use this as jobId when submitting feedback."+ },"links": {"type": "array","items": {@@ -2927,6 +3112,180 @@}}},+ "FeedbackErrorResponse": {+ "type": "object",+ "properties": {+ "success": {+ "type": "boolean",+ "example": false+ },+ "error": {+ "type": "string"+ },+ "feedbackErrorCode": {+ "type": "string"+ },+ "details": {+ "type": "array",+ "items": {+ "type": "object"+ }+ }+ },+ "required": ["success", "error"]+ },+ "FeedbackResponse": {+ "type": "object",+ "properties": {+ "success": {+ "type": "boolean",+ "example": true+ },+ "feedbackId": {+ "type": "string",+ "format": "uuid"+ },+ "creditsRefunded": {+ "type": "number"+ },+ "alreadySubmitted": {+ "type": "boolean"+ },+ "dailyCapReached": {+ "type": "boolean"+ },+ "creditsRefundedToday": {+ "type": "number"+ },+ "dailyRefundCap": {+ "type": "number"+ },+ "warning": {+ "type": "string"+ }+ },+ "required": ["success", "feedbackId", "creditsRefunded"]+ },+ "SearchFeedbackRequest": {+ "type": "object",+ "description": "For 'good', include valuableSources. For 'partial', include valuableSources or missingContent. For 'bad', include missingContent or querySuggestions.",+ "properties": {+ "rating": {+ "type": "string",+ "enum": ["good", "partial", "bad"]+ },+ "valuableSources": {+ "type": "array",+ "maxItems": 50,+ "items": {+ "type": "object",+ "properties": {+ "url": {+ "type": "string",+ "format": "uri"+ },+ "reason": {+ "type": "string",+ "maxLength": 1000+ }+ },+ "required": ["url"]+ }+ },+ "missingContent": {+ "type": "array",+ "maxItems": 20,+ "items": {+ "type": "object",+ "properties": {+ "topic": {+ "type": "string",+ "maxLength": 200+ },+ "description": {+ "type": "string",+ "maxLength": 2000+ }+ },+ "required": ["topic"]+ }+ },+ "querySuggestions": {+ "type": "string",+ "maxLength": 2000+ },+ "origin": {+ "type": "string",+ "default": "api"+ },+ "integration": {+ "type": "string",+ "nullable": true+ }+ },+ "required": ["rating"]+ },+ "EndpointFeedbackRequest": {+ "allOf": [+ {+ "$ref": "#/components/schemas/SearchFeedbackRequest"+ },+ {+ "type": "object",+ "description": "Submit feedback for a v2 job. Include at least one substantive signal such as issues, note, valuableSources, missingContent, querySuggestions, url, or pageNumbers.",+ "properties": {+ "endpoint": {+ "type": "string",+ "enum": ["search", "scrape", "parse", "map"]+ },+ "jobId": {+ "type": "string",+ "format": "uuid"+ },+ "issues": {+ "type": "array",+ "maxItems": 20,+ "items": {+ "type": "string",+ "pattern": "^[a-z0-9][a-z0-9_-]*$",+ "maxLength": 80+ }+ },+ "tags": {+ "type": "array",+ "maxItems": 20,+ "items": {+ "type": "string",+ "pattern": "^[a-z0-9][a-z0-9_-]*$",+ "maxLength": 80+ }+ },+ "note": {+ "type": "string",+ "maxLength": 4000+ },+ "url": {+ "type": "string",+ "format": "uri"+ },+ "pageNumbers": {+ "type": "array",+ "maxItems": 100,+ "items": {+ "type": "integer",+ "minimum": 1+ }+ },+ "metadata": {+ "type": "object",+ "additionalProperties": true,+ "description": "Small endpoint-specific metadata object. Must be 8KB or smaller; do not include full endpoint results."+ }+ },+ "required": ["endpoint", "jobId"]+ }+ ]+ },"ExtractResponse": {"type": "object","properties": {diff --git a/apps/api/src/__tests__/snips/index-cache.test.ts b/apps/api/src/__tests__/snips/index-cache.test.tsindex e1bb881eb..9d4366e41 100644--- a/apps/api/src/__tests__/snips/index-cache.test.ts+++ b/apps/api/src/__tests__/snips/index-cache.test.ts@@ -44,12 +44,12 @@ describe("deriveIndexVariantKey", () => {it("changes when any exact-match dimension changes", () => {const base = deriveIndexVariantKey(baseKeyParams);- expect(deriveIndexVariantKey({ ...baseKeyParams, isMobile: true })).not.toBe(- base,- );- expect(deriveIndexVariantKey({ ...baseKeyParams, blockAds: false })).not.toBe(- base,- );+ expect(+ deriveIndexVariantKey({ ...baseKeyParams, isMobile: true }),+ ).not.toBe(base);+ expect(+ deriveIndexVariantKey({ ...baseKeyParams, blockAds: false }),+ ).not.toBe(base);expect(deriveIndexVariantKey({ ...baseKeyParams, isStealth: true }),).not.toBe(base);@@ -209,9 +209,9 @@ describe("isNegativeStillValid", () => {it("is invalid for a larger window that looks further back than confirmed", () => {// 14-day window now reaches earlier than the 2-day confirmed-empty left edge.- expect(- isNegativeStillValid(emptyFrom, 14 * 24 * 60 * 60 * 1000, Tq),- ).toBe(false);+ expect(isNegativeStillValid(emptyFrom, 14 * 24 * 60 * 60 * 1000, Tq)).toBe(+ false,+ );});it("is exactly valid at the boundary", () => {diff --git a/apps/api/src/__tests__/snips/v2/endpoint-feedback.test.ts b/apps/api/src/__tests__/snips/v2/endpoint-feedback.test.tsnew file mode 100644index 000000000..48ad1e233--- /dev/null+++ b/apps/api/src/__tests__/snips/v2/endpoint-feedback.test.ts@@ -0,0 +1,164 @@+import { describeIf, TEST_PRODUCTION, TEST_SUITE_WEBSITE } from "../lib";+import {+ endpointFeedback,+ endpointFeedbackRaw,+ endpointFeedbackWithFailure,+ expectMapToSucceed,+ idmux,+ Identity,+ map,+ scrapeTimeout,+ searchRawFull,+} from "./lib";+import { and, eq } from "drizzle-orm";+import { db } from "../../../db/connection";+import * as schema from "../../../db/schema";++let identity: Identity;+let secondaryIdentity: Identity;++beforeAll(async () => {+ identity = await idmux({+ name: "endpoint-feedback",+ concurrency: 100,+ credits: 1000000,+ });+ secondaryIdentity = await idmux({+ name: "endpoint-feedback-other",+ concurrency: 100,+ credits: 1000000,+ });+}, 20000);++// Skipped in self-hosted mode: depends on the production job log tables,+// Autumn refunds, and the per-team daily refund cap.+describeIf(TEST_PRODUCTION)("Generic endpoint feedback tests", () => {+ it("records map feedback, exposes the map job id, and makes duplicate submissions idempotent", async () => {+ const raw = await map(+ { url: TEST_SUITE_WEBSITE, limit: 1, timeout: scrapeTimeout },+ identity,+ );+ expectMapToSucceed(raw);+ expect(typeof raw.body.id).toBe("string");++ const first = await endpointFeedback(+ {+ endpoint: "map",+ jobId: raw.body.id,+ rating: "bad",+ issues: ["missing_expected_url"],+ note: "The map result did not include the canonical page I expected.",+ },+ identity,+ );++ expect(first.success).toBe(true);+ expect(first.creditsRefunded).toBe(1);+ expect(first.alreadySubmitted).toBeFalsy();++ const second = await endpointFeedback(+ {+ endpoint: "map",+ jobId: raw.body.id,+ rating: "partial",+ issues: ["still_missing_expected_url"],+ note: "Submitting twice should not refund twice.",+ },+ identity,+ );++ expect(second.success).toBe(true);+ expect(second.creditsRefunded).toBe(0);+ expect(second.alreadySubmitted).toBe(true);+ }, 120000);++ it("rejects endpoint feedback for a job owned by another team", async () => {+ const raw = await map(+ { url: TEST_SUITE_WEBSITE, limit: 1, timeout: scrapeTimeout },+ identity,+ );+ expectMapToSucceed(raw);++ const failed = await endpointFeedbackWithFailure(+ {+ endpoint: "map",+ jobId: raw.body.id,+ rating: "bad",+ note: "This team should not be able to see the job.",+ },+ secondaryIdentity,+ );++ expect(failed.error.toLowerCase()).toContain("not found");+ expect((failed as any).feedbackErrorCode).toBe("JOB_NOT_FOUND");+ }, 120000);++ it("applies search feedback validation on the generic endpoint", async () => {+ const raw = await searchRawFull(+ { query: "firecrawl generic feedback", limit: 3 },+ identity,+ );+ expect(raw.statusCode).toBe(200);+ expect(typeof raw.body.id).toBe("string");++ const failed = await endpointFeedbackRaw(+ {+ endpoint: "search",+ jobId: raw.body.id,+ rating: "good",+ note: "A good search rating must name a valuable source.",+ },+ identity,+ );++ expect(failed.statusCode).toBe(400);+ expect(failed.body.success).toBe(false);+ expect(String(failed.body.error).toLowerCase()).toContain("invalid");+ }, 90000);++ it("rejects generic search feedback for failed search jobs", async () => {+ const searchId =+ "00000000-0000-7000-8000-" ++ Math.floor(Math.random() * 1e12)+ .toString(16)+ .padStart(12, "0");++ await db.insert(schema.searches).values({+ id: searchId,+ request_id: searchId,+ query: "failed generic feedback search",+ team_id: identity.teamId,+ options: { query: "failed generic feedback search" },+ time_taken: 0,+ credits_cost: 2,+ is_successful: false,+ error: "Synthetic failed search for generic feedback policy coverage.",+ num_results: 0,+ });++ try {+ const failed = await endpointFeedbackRaw(+ {+ endpoint: "search",+ jobId: searchId,+ rating: "bad",+ missingContent: [{ topic: "Results" }],+ },+ identity,+ );++ expect(failed.statusCode).toBe(409);+ expect(failed.body.success).toBe(false);+ expect(failed.body.feedbackErrorCode).toBe("SEARCH_FAILED");+ } finally {+ await db+ .delete(schema.searches)+ .where(+ and(+ eq(schema.searches.id, searchId),+ eq(schema.searches.team_id, identity.teamId),+ ),+ );+ }+ }, 30000);+});diff --git a/apps/api/src/__tests__/snips/v2/lib.ts b/apps/api/src/__tests__/snips/v2/lib.tsindex 14f392082..0ed0e701e 100644--- a/apps/api/src/__tests__/snips/v2/lib.ts+++ b/apps/api/src/__tests__/snips/v2/lib.ts@@ -14,6 +14,8 @@ import {BatchScrapeRequestInput,SearchRequestInput,SearchFeedbackRequestInput,+ EndpointFeedbackRequestInput,+ EndpointFeedbackResponse,ParseRequestInput,} from "../../../controllers/v2/types";import request from "supertest";@@ -703,6 +705,54 @@ export async function searchFeedbackWithFailure(return raw.body;}+// =========================================+// Generic Feedback API+// =========================================++export async function endpointFeedbackRaw(+ body: EndpointFeedbackRequestInput,+ identity: Identity,+) {+ return await request(TEST_API_URL)+ .post("/v2/feedback")+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .set("Content-Type", "application/json")+ .send(body);+}++export async function endpointFeedback(+ body: EndpointFeedbackRequestInput,+ identity: Identity,+): Promise<Exclude<EndpointFeedbackResponse, ErrorResponse>> {+ const raw = await endpointFeedbackRaw(body, identity);+ if (raw.statusCode !== 200) {+ console.warn(+ "Endpoint feedback did not succeed",+ JSON.stringify(raw.body, null, 2),+ );+ }+ expect(raw.statusCode).toBe(200);+ expect(raw.body.success).toBe(true);+ expect(typeof raw.body.feedbackId).toBe("string");+ expect(typeof raw.body.creditsRefunded).toBe("number");+ return raw.body;+}++export async function endpointFeedbackWithFailure(+ body: EndpointFeedbackRequestInput,+ identity: Identity,+): Promise<{+ success: false;+ error: string;+ details?: unknown;+}> {+ const raw = await endpointFeedbackRaw(body, identity);+ expect(raw.statusCode).not.toBe(200);+ expect(raw.body.success).toBe(false);+ expect(typeof raw.body.error).toBe("string");+ return raw.body;+}+// =========================================// Billing API// =========================================diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 06abdadf7..454c946d5 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -80,6 +80,9 @@ const configSchema = z.object({.int().nonnegative().default(100),+ FEEDBACK_MAX_AGE_SEC: z.coerce.number().int().positive().default(120),+ FEEDBACK_DAILY_CAP_CREDITS: z.coerce.number().int().nonnegative().default(50),+ FEEDBACK_REFUND_ENABLED: z.stringbool().default(true),// OAuth token introspectionOAUTH_INTROSPECT_URL: z.string().optional(),diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.tsindex cae21d208..7c1db8efd 100644--- a/apps/api/src/controllers/v2/crawl-status.ts+++ b/apps/api/src/controllers/v2/crawl-status.ts@@ -374,16 +374,15 @@ export async function crawlStatusController(const status = outputBulkA.status ?? "scraping";const createdAtMs = sc?.createdAt;const lastDoneMs =- status !== "scraping" ? await getLastDoneJobTimestamp(req.params.jobId) : null;+ status !== "scraping"+ ? await getLastDoneJobTimestamp(req.params.jobId)+ : null;const completedAtMs =status === "completed" || status === "failed" || status === "cancelled"- ? lastDoneMs ?? createdAtMs ?? null+ ? (lastDoneMs ?? createdAtMs ?? null): null;const durationSeconds = createdAtMs- ? Math.max(- 0,- ((completedAtMs ?? Date.now()) - createdAtMs) / 1000,- )+ ? Math.max(0, ((completedAtMs ?? Date.now()) - createdAtMs) / 1000): undefined;return res.status(200).json({@@ -396,7 +395,9 @@ export async function crawlStatusController(next: outputBulkB.next,data: outputBulkB.data,...(createdAtMs && { createdAt: new Date(createdAtMs).toISOString() }),- ...(completedAtMs && { completedAt: new Date(completedAtMs).toISOString() }),+ ...(completedAtMs && {+ completedAt: new Date(completedAtMs).toISOString(),+ }),...(durationSeconds !== undefined && { duration: durationSeconds }),...(warning && { warning }),});diff --git a/apps/api/src/controllers/v2/feedback/controller.ts b/apps/api/src/controllers/v2/feedback/controller.tsnew file mode 100644index 000000000..1114aed0e--- /dev/null+++ b/apps/api/src/controllers/v2/feedback/controller.ts@@ -0,0 +1,42 @@+import { Response } from "express";+import { z } from "zod";+import {+ EndpointFeedbackRequest,+ EndpointFeedbackResponse,+ RequestWithAuth,+ endpointFeedbackSchema,+} from "../types";+import { recordEndpointFeedback } from "./record";+import { endpointFeedbackRecordOptions } from "./record-options";+import { toFeedbackInput } from "./request-input";++export async function feedbackController(+ req: RequestWithAuth<{}, EndpointFeedbackResponse, EndpointFeedbackRequest>,+ res: Response<EndpointFeedbackResponse>,+) {+ let parsedBody: EndpointFeedbackRequest;+ try {+ parsedBody = endpointFeedbackSchema.parse(req.body);+ } catch (error) {+ if (error instanceof z.ZodError) {+ return res.status(400).json({+ success: false,+ error: "Invalid request body",+ details: error.issues,+ feedbackErrorCode: "INVALID_BODY",+ });+ }+ throw error;+ }++ const result = await recordEndpointFeedback(+ req,+ endpointFeedbackRecordOptions({+ endpoint: parsedBody.endpoint,+ jobId: parsedBody.jobId,+ feedback: toFeedbackInput(parsedBody),+ }),+ );++ return res.status(result.status).json(result.body);+}diff --git a/apps/api/src/controllers/v2/feedback/feedback-store.ts b/apps/api/src/controllers/v2/feedback/feedback-store.tsnew file mode 100644index 000000000..67ef99d5e--- /dev/null+++ b/apps/api/src/controllers/v2/feedback/feedback-store.ts@@ -0,0 +1,185 @@+import { and, eq } from "drizzle-orm";+import { db, dbRr } from "../../../db/connection";+import * as schema from "../../../db/schema";+import { EndpointFeedbackEndpoint } from "../types";+import {+ FeedbackJobRow,+ FeedbackRecordOptions,+ RefundPolicySnapshot,+} from "./internal-types";++type DbError = { code?: string } & Record<string, unknown>;++type ExistingFeedback = {+ id: string;+ credits_refunded: number | null;+};++const JOB_TABLES = {+ search: schema.searches,+ scrape: schema.scrapes,+ parse: schema.parses,+ map: schema.maps,+} as const;++function feedbackMetadata(+ options: FeedbackRecordOptions,+): Record<string, unknown> {+ return {+ ...(options.feedback.metadata ?? {}),+ ...(options.feedback.url ? { url: options.feedback.url } : {}),+ ...(options.feedback.pageNumbers+ ? { pageNumbers: options.feedback.pageNumbers }+ : {}),+ };+}++export async function lookupFeedbackJob(+ endpoint: EndpointFeedbackEndpoint,+ jobId: string,+ dbTeamId: string,+): Promise<FeedbackJobRow | null> {+ const table = JOB_TABLES[endpoint] as any;+ const [row] = await dbRr+ .select({+ id: table.id,+ request_id: table.request_id,+ team_id: table.team_id,+ credits_cost: table.credits_cost,+ created_at: table.created_at,+ options: table.options,+ ...(endpoint === "map" ? {} : { is_successful: table.is_successful }),+ })+ .from(table)+ .where(and(eq(table.id, jobId), eq(table.team_id, dbTeamId)))+ .limit(1);++ if (!row) return null;++ return {+ endpoint,+ id: row.id,+ request_id: row.request_id ?? null,+ team_id: row.team_id,+ credits_cost: row.credits_cost ?? 0,+ created_at: row.created_at,+ is_successful: endpoint === "map" ? true : (row.is_successful ?? null),+ options: row.options ?? null,+ };+}++export async function insertFeedback(params: {+ feedbackId: string;+ options: FeedbackRecordOptions;+ job: FeedbackJobRow;+ dbTeamId: string;+ apiKeyId?: number | null;+}): Promise<DbError | null> {+ const { feedbackId, options, job, dbTeamId, apiKeyId } = params;+ try {+ await db.insert(schema.search_feedback).values({+ id: feedbackId,+ search_id: options.endpoint === "search" ? options.jobId : null,+ endpoint: options.endpoint,+ job_id: options.jobId,+ request_id: job.request_id,+ api_version: "v2",+ team_id: dbTeamId,+ api_key_id: apiKeyId ?? null,+ overall_rating: options.feedback.rating,+ issue_types: options.feedback.issues ?? [],+ tags: options.feedback.tags ?? [],+ comment: options.feedback.note ?? null,+ valuable_sources: options.feedback.valuableSources ?? [],+ missing_content: options.feedback.missingContent ?? [],+ query_suggestions: options.feedback.querySuggestions ?? null,+ metadata: feedbackMetadata(options),+ job_status: job.is_successful === false ? "failed" : "completed",+ credits_billed: job.credits_cost ?? 0,+ credits_refunded: 0,+ refund_policy: null,+ integration: options.feedback.integration ?? null,+ origin: options.feedback.origin ?? null,+ });+ return null;+ } catch (error) {+ return error as DbError;+ }+}++async function findFeedbackByJob(+ dbTeamId: string,+ endpoint: EndpointFeedbackEndpoint,+ jobId: string,+): Promise<ExistingFeedback | null> {+ const [row] = await dbRr+ .select({+ id: schema.search_feedback.id,+ credits_refunded: schema.search_feedback.credits_refunded,+ })+ .from(schema.search_feedback)+ .where(+ and(+ eq(schema.search_feedback.team_id, dbTeamId),+ eq(schema.search_feedback.endpoint, endpoint),+ eq(schema.search_feedback.job_id, jobId),+ ),+ )+ .limit(1);++ return row ?? null;+}++async function findSearchFeedbackByLegacyId(+ dbTeamId: string,+ searchId: string,+): Promise<ExistingFeedback | null> {+ const [row] = await dbRr+ .select({+ id: schema.search_feedback.id,+ credits_refunded: schema.search_feedback.credits_refunded,+ })+ .from(schema.search_feedback)+ .where(+ and(+ eq(schema.search_feedback.team_id, dbTeamId),+ eq(schema.search_feedback.search_id, searchId),+ ),+ )+ .limit(1);++ return row ?? null;+}++export async function findExistingFeedback(+ dbTeamId: string,+ endpoint: EndpointFeedbackEndpoint,+ jobId: string,+): Promise<ExistingFeedback | null> {+ return (+ (await findFeedbackByJob(dbTeamId, endpoint, jobId)) ??+ (endpoint === "search"+ ? await findSearchFeedbackByLegacyId(dbTeamId, jobId)+ : null)+ );+}++export async function updateFeedbackRefundDetails(+ feedbackId: string,+ creditsRefunded: number,+ policy: RefundPolicySnapshot,+): Promise<DbError | null> {+ try {+ await db+ .update(schema.search_feedback)+ .set({+ credits_refunded: creditsRefunded,+ refund_policy: policy,+ updated_at: new Date().toISOString(),+ })+ .where(eq(schema.search_feedback.id, feedbackId));+ return null;+ } catch (error) {+ return error as DbError;+ }+}diff --git a/apps/api/src/controllers/v2/feedback/internal-types.ts b/apps/api/src/controllers/v2/feedback/internal-types.tsnew file mode 100644index 000000000..e75bd95e0--- /dev/null+++ b/apps/api/src/controllers/v2/feedback/internal-types.ts@@ -0,0 +1,70 @@+import { logger as _logger } from "../../../lib/logger";+import {+ EndpointFeedbackEndpoint,+ EndpointFeedbackErrorCode,+ EndpointFeedbackResponse,+ SearchFeedbackErrorCode,+} from "../types";++export type FeedbackRating = "good" | "partial" | "bad";++export type FeedbackInput = {+ rating: FeedbackRating;+ issues?: string[];+ tags?: string[];+ note?: string;+ valuableSources?: Array<{ url: string; reason?: string }>;+ missingContent?: Array<{ topic: string; description?: string }>;+ querySuggestions?: string;+ url?: string;+ pageNumbers?: number[];+ metadata?: Record<string, unknown>;+ origin?: string;+ integration?: string | null;+};++export type FeedbackJobRow = {+ endpoint: EndpointFeedbackEndpoint;+ id: string;+ request_id: string | null;+ team_id: string;+ credits_cost: number | null;+ created_at: string;+ is_successful: boolean | null;+ options: unknown;+};++export type FeedbackRecordOptions = {+ endpoint: EndpointFeedbackEndpoint;+ jobId: string;+ feedback: FeedbackInput;+ requireSuccessfulJob?: boolean;+ notFoundCode?: EndpointFeedbackErrorCode | SearchFeedbackErrorCode;+ failedJobCode?: SearchFeedbackErrorCode;+ dbDisabledMessage?: string;+ windowExpiredMessage?: string;+ maxAgeSec?: number;+ dailyCapCredits?: number;+ refundFeatureId?: string;+ skipZdrPersistence?: boolean;+ source: "feedback" | "search_feedback";+};++export type FeedbackRecordResult = {+ status: number;+ body: EndpointFeedbackResponse | any;+};++export type RefundPolicySnapshot = {+ version: "feedback_refund_v1";+ enabled: boolean;+ endpoint: EndpointFeedbackEndpoint;+ mode: "none" | "flat" | "percentage_with_cap";+ refundableRatings: FeedbackRating[];+ matchedReason: string;+ flatCredits?: number;+ percent?: number;+ maxCredits?: number;+};++export type FeedbackLogger = ReturnType<typeof _logger.child>;diff --git a/apps/api/src/controllers/v2/feedback/record-options.test.ts b/apps/api/src/controllers/v2/feedback/record-options.test.tsnew file mode 100644index 000000000..477dc5c52--- /dev/null+++ b/apps/api/src/controllers/v2/feedback/record-options.test.ts@@ -0,0 +1,118 @@+import { config } from "../../../config";+import { SEARCH_CREDITS_FEATURE_ID } from "../../../services/autumn/autumn.service";+import { endpointFeedbackSchema, searchFeedbackSchema } from "../types";+import {+ endpointFeedbackRecordOptions,+ searchFeedbackRecordOptions,+} from "./record-options";++const originalConfig = {+ SEARCH_FEEDBACK_MAX_AGE_SEC: config.SEARCH_FEEDBACK_MAX_AGE_SEC,+ SEARCH_FEEDBACK_DAILY_CAP_CREDITS: config.SEARCH_FEEDBACK_DAILY_CAP_CREDITS,+ FEEDBACK_MAX_AGE_SEC: config.FEEDBACK_MAX_AGE_SEC,+ FEEDBACK_DAILY_CAP_CREDITS: config.FEEDBACK_DAILY_CAP_CREDITS,+};++afterEach(() => {+ Object.assign(config, originalConfig);+});++describe("feedback record options", () => {+ it("applies the search feedback policy to legacy search feedback", () => {+ config.SEARCH_FEEDBACK_MAX_AGE_SEC = 17;+ config.SEARCH_FEEDBACK_DAILY_CAP_CREDITS = 3;+ config.FEEDBACK_MAX_AGE_SEC = 999;+ config.FEEDBACK_DAILY_CAP_CREDITS = 999;++ const options = searchFeedbackRecordOptions({+ jobId: "01933161-0000-7000-8000-000000000001",+ feedback: { rating: "bad", note: "Bad search result" },+ });++ expect(options.endpoint).toBe("search");+ expect(options.requireSuccessfulJob).toBe(true);+ expect(options.notFoundCode).toBe("SEARCH_NOT_FOUND");+ expect(options.failedJobCode).toBe("SEARCH_FAILED");+ expect(options.maxAgeSec).toBe(17);+ expect(options.dailyCapCredits).toBe(3);+ expect(options.refundFeatureId).toBe(SEARCH_CREDITS_FEATURE_ID);+ expect(options.source).toBe("search_feedback");+ });++ it("applies the search feedback policy to generic search feedback", () => {+ config.SEARCH_FEEDBACK_MAX_AGE_SEC = 29;+ config.SEARCH_FEEDBACK_DAILY_CAP_CREDITS = 7;+ config.FEEDBACK_MAX_AGE_SEC = 999;+ config.FEEDBACK_DAILY_CAP_CREDITS = 999;++ const options = endpointFeedbackRecordOptions({+ endpoint: "search",+ jobId: "01933161-0000-7000-8000-000000000001",+ feedback: { rating: "bad", note: "Bad search result" },+ });++ expect(options.endpoint).toBe("search");+ expect(options.requireSuccessfulJob).toBe(true);+ expect(options.notFoundCode).toBe("SEARCH_NOT_FOUND");+ expect(options.failedJobCode).toBe("SEARCH_FAILED");+ expect(options.maxAgeSec).toBe(29);+ expect(options.dailyCapCredits).toBe(7);+ expect(options.refundFeatureId).toBe(SEARCH_CREDITS_FEATURE_ID);+ expect(options.source).toBe("search_feedback");+ });++ it("keeps generic non-search feedback on the generic policy", () => {+ const options = endpointFeedbackRecordOptions({+ endpoint: "map",+ jobId: "01933161-0000-7000-8000-000000000001",+ feedback: { rating: "bad", note: "Bad map result" },+ });++ expect(options.endpoint).toBe("map");+ expect(options.requireSuccessfulJob).toBeUndefined();+ expect(options.notFoundCode).toBeUndefined();+ expect(options.failedJobCode).toBeUndefined();+ expect(options.maxAgeSec).toBeUndefined();+ expect(options.dailyCapCredits).toBeUndefined();+ expect(options.refundFeatureId).toBeUndefined();+ expect(options.source).toBe("feedback");+ });+});++describe("feedback schema", () => {+ it("keeps generic search feedback on the search-specific rating rules", () => {+ expect(+ searchFeedbackSchema.safeParse({+ rating: "good",+ valuableSources: [{ url: "https://firecrawl.dev/" }],+ }).success,+ ).toBe(true);++ expect(+ endpointFeedbackSchema.safeParse({+ endpoint: "search",+ jobId: "01933161-0000-7000-8000-000000000001",+ rating: "good",+ valuableSources: [{ url: "https://firecrawl.dev/" }],+ }).success,+ ).toBe(true);++ expect(+ endpointFeedbackSchema.safeParse({+ endpoint: "search",+ jobId: "01933161-0000-7000-8000-000000000001",+ rating: "good",+ note: "A note alone should not satisfy search feedback rules.",+ }).success,+ ).toBe(false);++ expect(+ endpointFeedbackSchema.safeParse({+ endpoint: "map",+ jobId: "01933161-0000-7000-8000-000000000001",+ rating: "good",+ note: "Generic endpoint feedback remains valid with a note.",+ }).success,+ ).toBe(true);+ });+});diff --git a/apps/api/src/controllers/v2/feedback/record-options.ts b/apps/api/src/controllers/v2/feedback/record-options.tsnew file mode 100644index 000000000..10dcd1bbb--- /dev/null+++ b/apps/api/src/controllers/v2/feedback/record-options.ts@@ -0,0 +1,53 @@+import { config } from "../../../config";+import { SEARCH_CREDITS_FEATURE_ID } from "../../../services/autumn/autumn.service";+import type { EndpointFeedbackEndpoint } from "../types";+import type { FeedbackInput, FeedbackRecordOptions } from "./internal-types";++type SearchFeedbackRecordOptionsParams = {+ jobId: string;+ feedback: FeedbackInput;+};++type EndpointFeedbackRecordOptionsParams = {+ endpoint: EndpointFeedbackEndpoint;+ jobId: string;+ feedback: FeedbackInput;+};++export function searchFeedbackRecordOptions({+ jobId,+ feedback,+}: SearchFeedbackRecordOptionsParams): FeedbackRecordOptions {+ return {+ endpoint: "search",+ jobId,+ feedback,+ requireSuccessfulJob: true,+ notFoundCode: "SEARCH_NOT_FOUND",+ failedJobCode: "SEARCH_FAILED",+ dbDisabledMessage:+ "Search feedback requires database authentication and is unavailable on this deployment.",+ windowExpiredMessage: `Search feedback must be submitted within ${config.SEARCH_FEEDBACK_MAX_AGE_SEC} seconds of the search.`,+ maxAgeSec: config.SEARCH_FEEDBACK_MAX_AGE_SEC,+ dailyCapCredits: config.SEARCH_FEEDBACK_DAILY_CAP_CREDITS,+ refundFeatureId: SEARCH_CREDITS_FEATURE_ID,+ source: "search_feedback",+ };+}++export function endpointFeedbackRecordOptions({+ endpoint,+ jobId,+ feedback,+}: EndpointFeedbackRecordOptionsParams): FeedbackRecordOptions {+ if (endpoint === "search") {+ return searchFeedbackRecordOptions({ jobId, feedback });+ }++ return {+ endpoint,+ jobId,+ feedback,+ source: "feedback",+ };+}diff --git a/apps/api/src/controllers/v2/feedback/record.ts b/apps/api/src/controllers/v2/feedback/record.tsnew file mode 100644index 000000000..63c262a6c--- /dev/null+++ b/apps/api/src/controllers/v2/feedback/record.ts@@ -0,0 +1,375 @@+import { v7 as uuidv7 } from "uuid";+import { config } from "../../../config";+import { logger as _logger } from "../../../lib/logger";+import {+ autumnService,+ featureIdForBillingEndpoint,+} from "../../../services/autumn/autumn.service";+import { captureExceptionWithZdrCheck } from "../../../services/sentry";+import {+ EndpointFeedbackErrorCode,+ RequestWithAuth,+ SearchFeedbackErrorCode,+} from "../types";+import {+ findExistingFeedback,+ insertFeedback,+ lookupFeedbackJob,+ updateFeedbackRefundDetails,+} from "./feedback-store";+import {+ FeedbackJobRow,+ FeedbackLogger,+ FeedbackRecordOptions,+ FeedbackRecordResult,+ RefundPolicySnapshot,+} from "./internal-types";+import { computeRefundPolicy } from "./refund-policy";+import { sumCreditsRefundedToday } from "./refund-totals";+import {+ shouldSkipPersistenceForForcedZdr,+ shouldSkipPersistenceForJobZdr,+} from "./zdr-persistence";++const PREVIEW_TEAM_ID = "3adefd26-77ec-5968-8dcf-c94b5630d1de";+const POSTGRES_UNIQUE_VIOLATION = "23505";+const LOOKUP_RACE_RETRY_MS = 250;+const ZDR_FEEDBACK_ID = "00000000-0000-0000-0000-000000000000";++function isPreviewTeam(teamId: string): boolean {+ return teamId === "preview" || teamId.startsWith("preview_");+}++function normalizeFeedbackTeamId(teamId: string): string {+ return isPreviewTeam(teamId) ? PREVIEW_TEAM_ID : teamId;+}++function dailyCapFor(options: FeedbackRecordOptions): number {+ return options.dailyCapCredits ?? config.FEEDBACK_DAILY_CAP_CREDITS;+}++function feedbackFailure(+ status: number,+ code: EndpointFeedbackErrorCode | SearchFeedbackErrorCode,+ error: string,+): FeedbackRecordResult {+ return {+ status,+ body: {+ success: false,+ error,+ feedbackErrorCode: code,+ },+ };+}++function zdrFeedbackSuccess(+ options: FeedbackRecordOptions,+): FeedbackRecordResult {+ return {+ status: 200,+ body: {+ success: true,+ feedbackId: ZDR_FEEDBACK_ID,+ creditsRefunded: 0,+ creditsRefundedToday: 0,+ dailyRefundCap: dailyCapFor(options),+ },+ };+}++function validateAccess(+ req: RequestWithAuth<any, any, any>,+ options: FeedbackRecordOptions,+ logger: FeedbackLogger,+): FeedbackRecordResult | null {+ if (config.USE_DB_AUTHENTICATION !== true) {+ return feedbackFailure(+ 503,+ "DB_DISABLED",+ options.dbDisabledMessage ??+ "Feedback requires database authentication and is unavailable on this deployment.",+ );+ }++ if (isPreviewTeam(req.auth.team_id)) {+ return feedbackFailure(+ 403,+ "PREVIEW_TEAM_NOT_ALLOWED",+ "Feedback is not available for preview teams.",+ );+ }++ if (req.acuc?.flags?.searchFeedbackOptOut === true) {+ logger.info("Rejected feedback: team opted out");+ return feedbackFailure(+ 403,+ "TEAM_OPTED_OUT",+ "Feedback is disabled for this team. Contact [email protected] to re-enable.",+ );+ }++ return null;+}++async function lookupJobWithRetry(+ options: FeedbackRecordOptions,+ dbTeamId: string,+ logger: FeedbackLogger,+): Promise<FeedbackJobRow | FeedbackRecordResult> {+ try {+ let job = await lookupFeedbackJob(+ options.endpoint,+ options.jobId,+ dbTeamId,+ );+ if (!job) {+ await new Promise(resolve => setTimeout(resolve, LOOKUP_RACE_RETRY_MS));+ job = await lookupFeedbackJob(options.endpoint, options.jobId, dbTeamId);+ }++ if (!job) {+ return feedbackFailure(+ 404,+ options.notFoundCode ?? "JOB_NOT_FOUND",+ `${options.endpoint} job not found for this team.`,+ );+ }++ return job;+ } catch (error) {+ logger.error("Failed to look up job for feedback", { error });+ return feedbackFailure(500, "INTERNAL", "Failed to look up job.");+ }+}++function validateJob(+ job: FeedbackJobRow,+ options: FeedbackRecordOptions,+ logger: FeedbackLogger,+): FeedbackRecordResult | null {+ if (options.requireSuccessfulJob && job.is_successful === false) {+ return feedbackFailure(+ 409,+ options.failedJobCode ?? "INTERNAL",+ `Cannot submit feedback for a ${options.endpoint} job that did not succeed.`,+ );+ }++ const maxAgeSec = options.maxAgeSec ?? config.FEEDBACK_MAX_AGE_SEC;+ const createdAtMs = new Date(job.created_at).getTime();+ if (Number.isNaN(createdAtMs)) {+ logger.warn("Job row had unparseable created_at", {+ created_at: job.created_at,+ });+ return null;+ }++ if (Date.now() - createdAtMs <= maxAgeSec * 1000) return null;++ return feedbackFailure(+ 409,+ "FEEDBACK_WINDOW_EXPIRED",+ options.windowExpiredMessage ??+ `Feedback must be submitted within ${maxAgeSec} seconds of the job.`,+ );+}++async function duplicateEndpointResponse(+ options: FeedbackRecordOptions,+ dbTeamId: string,+ logger: FeedbackLogger,+): Promise<FeedbackRecordResult> {+ const existing = await findExistingFeedback(+ dbTeamId,+ options.endpoint,+ options.jobId,+ );++ return {+ status: 200,+ body: {+ success: true,+ feedbackId: existing?.id ?? "",+ creditsRefunded: 0,+ alreadySubmitted: true,+ creditsRefundedToday: await sumCreditsRefundedToday(dbTeamId, logger),+ dailyRefundCap: dailyCapFor(options),+ warning:+ "Feedback was already submitted for this job; no additional refund issued.",+ },+ };+}++async function refundCredits(params: {+ req: RequestWithAuth<any, any, any>;+ options: FeedbackRecordOptions;+ feedbackId: string;+ cappedRefund: number;+ policy: RefundPolicySnapshot;+ logger: FeedbackLogger;+}): Promise<number> {+ const { req, options, feedbackId, cappedRefund, policy, logger } = params;+ if (cappedRefund <= 0) return 0;++ try {+ await autumnService.refundCredits({+ teamId: req.auth.team_id,+ value: cappedRefund,+ featureId:+ options.refundFeatureId ??+ featureIdForBillingEndpoint(options.endpoint),+ properties: {+ source: options.source,+ endpoint: options.endpoint,+ jobId: options.jobId,+ feedbackId,+ rating: options.feedback.rating,+ refundPolicy: policy.matchedReason,+ },+ });+ return cappedRefund;+ } catch (error) {+ logger.error("Feedback refund failed; feedback retained", { error });+ return 0;+ }+}++export async function recordEndpointFeedback(+ req: RequestWithAuth<any, any, any>,+ options: FeedbackRecordOptions,+): Promise<FeedbackRecordResult> {+ const logger = _logger.child({+ module: "api/v2",+ method: "recordEndpointFeedback",+ endpoint: options.endpoint,+ jobId: options.jobId,+ teamId: req.auth.team_id,+ });++ if (shouldSkipPersistenceForForcedZdr(req, options)) {+ logger.info("Skipping feedback persistence for forced ZDR team");+ return zdrFeedbackSuccess(options);+ }++ const accessFailure = validateAccess(req, options, logger);+ if (accessFailure) return accessFailure;++ const dbTeamId = normalizeFeedbackTeamId(req.auth.team_id);++ try {+ const jobOrFailure = await lookupJobWithRetry(options, dbTeamId, logger);+ if ("status" in jobOrFailure) return jobOrFailure;++ if (shouldSkipPersistenceForJobZdr(jobOrFailure, options)) {+ logger.info("Skipping feedback persistence for ZDR job");+ return zdrFeedbackSuccess(options);+ }++ const jobFailure = validateJob(jobOrFailure, options, logger);+ if (jobFailure) return jobFailure;++ const feedbackId = uuidv7();+ const insertErr = await insertFeedback({+ feedbackId,+ options,+ job: jobOrFailure,+ dbTeamId,+ apiKeyId: req.acuc?.api_key_id ?? null,+ });++ if (insertErr) {+ if (insertErr.code === POSTGRES_UNIQUE_VIOLATION) {+ return await duplicateEndpointResponse(options, dbTeamId, logger);+ }++ logger.error("Failed to insert endpoint feedback", { error: insertErr });+ return feedbackFailure(500, "INTERNAL", "Failed to record feedback.");+ }++ const dailyCap = dailyCapFor(options);+ const refundedTodayBefore = await sumCreditsRefundedToday(dbTeamId, logger);+ const { desiredRefund, policy } = computeRefundPolicy(+ jobOrFailure,+ options.feedback.rating,+ );+ const cappedRefund = Math.min(+ desiredRefund,+ Math.max(0, dailyCap - refundedTodayBefore),+ );++ let dailyCapReached = false;+ if (desiredRefund > 0 && cappedRefund === 0) {+ dailyCapReached = true;+ logger.info(+ "Daily refund cap reached for team; feedback recorded with zero refund",+ { dailyCap, refundedTodayBefore },+ );+ }++ const creditsRefunded = await refundCredits({+ req,+ options,+ feedbackId,+ cappedRefund,+ policy,+ logger,+ });++ const updateErr = await updateFeedbackRefundDetails(+ feedbackId,+ creditsRefunded,+ policy,+ );+ if (updateErr) {+ logger.warn("Failed to persist endpoint feedback refund details", {+ error: updateErr,+ feedbackId,+ creditsRefunded,+ });+ }++ const creditsRefundedToday = refundedTodayBefore + creditsRefunded;+ dailyCapReached ||= creditsRefundedToday >= dailyCap && dailyCap > 0;++ logger.info("Endpoint feedback recorded", {+ feedbackId,+ endpoint: options.endpoint,+ creditsRefunded,+ creditsBilled: jobOrFailure.credits_cost ?? 0,+ rating: options.feedback.rating,+ issueTypes: options.feedback.issues ?? [],+ refundPolicy: policy.matchedReason,+ creditsRefundedToday,+ dailyRefundCap: dailyCap,+ dailyCapReached,+ });++ return {+ status: 200,+ body: {+ success: true,+ feedbackId,+ creditsRefunded,+ creditsRefundedToday,+ dailyRefundCap: dailyCap,+ ...(dailyCapReached+ ? {+ dailyCapReached: true,+ warning: `Daily refund cap of ${dailyCap} credits reached for this team (UTC day). Feedback was recorded; further /feedback calls today will not refund credits.`,+ }+ : {}),+ },+ };+ } catch (error) {+ captureExceptionWithZdrCheck(error);+ logger.error("Unhandled error while recording endpoint feedback", {+ error,+ });+ return feedbackFailure(+ 500,+ "INTERNAL",+ error instanceof Error ? error.message : "Unknown error",+ );+ }+}diff --git a/apps/api/src/controllers/v2/feedback/refund-policy.ts b/apps/api/src/controllers/v2/feedback/refund-policy.tsnew file mode 100644index 000000000..cf0e557f4--- /dev/null+++ b/apps/api/src/controllers/v2/feedback/refund-policy.ts@@ -0,0 +1,170 @@+import { config } from "../../../config";+import {+ FeedbackJobRow,+ FeedbackRating,+ RefundPolicySnapshot,+} from "./internal-types";++type RefundablePolicy =+ | {+ mode: "flat";+ reason: string;+ ratings: FeedbackRating[];+ credits: number;+ }+ | {+ mode: "percentage_with_cap";+ reason: string;+ ratings: FeedbackRating[];+ percent: number;+ maxCredits: number;+ };++function optionListIncludes(+ options: unknown,+ key: string,+ value: string,+): boolean {+ const list = (options as Record<string, unknown> | null)?.[key];+ if (!Array.isArray(list)) return false;++ return list.some(item => {+ if (item === value) return true;+ return (+ !!item &&+ typeof item === "object" &&+ (item as { type?: unknown }).type === value+ );+ });+}++function hasActions(options: unknown): boolean {+ const actions = (options as { actions?: unknown } | null)?.actions;+ return Array.isArray(actions) && actions.length > 0;+}++function none(+ job: FeedbackJobRow,+ matchedReason: string,+ refundableRatings: FeedbackRating[] = [],+): { desiredRefund: number; policy: RefundPolicySnapshot } {+ return {+ desiredRefund: 0,+ policy: {+ version: "feedback_refund_v1",+ enabled: config.FEEDBACK_REFUND_ENABLED,+ endpoint: job.endpoint,+ mode: "none",+ refundableRatings,+ matchedReason,+ },+ };+}++function policyFor(job: FeedbackJobRow): RefundablePolicy {+ switch (job.endpoint) {+ case "search":+ return {+ mode: "flat",+ reason: "search_feedback",+ ratings: ["good", "partial", "bad"],+ credits: 1,+ };+ case "map":+ return {+ mode: "flat",+ reason: "map_feedback",+ ratings: ["partial", "bad"],+ credits: 1,+ };+ case "parse":+ return {+ mode: "percentage_with_cap",+ reason: "parse_feedback",+ ratings: ["partial", "bad"],+ percent: 0.25,+ maxCredits: 10,+ };+ case "scrape":+ if (optionListIncludes(job.options, "parsers", "pdf")) {+ return {+ mode: "percentage_with_cap",+ reason: "scrape_pdf_feedback",+ ratings: ["partial", "bad"],+ percent: 0.25,+ maxCredits: 10,+ };+ }++ if (+ optionListIncludes(job.options, "formats", "json") ||+ optionListIncludes(job.options, "formats", "screenshot") ||+ hasActions(job.options)+ ) {+ return {+ mode: "percentage_with_cap",+ reason: optionListIncludes(job.options, "formats", "json")+ ? "scrape_json_feedback"+ : "scrape_addon_feedback",+ ratings: ["partial", "bad"],+ percent: 0.25,+ maxCredits: 5,+ };+ }++ return {+ mode: "flat",+ reason: "scrape_feedback",+ ratings: ["partial", "bad"],+ credits: 1,+ };+ }+}++export function computeRefundPolicy(+ job: FeedbackJobRow,+ rating: FeedbackRating,+): { desiredRefund: number; policy: RefundPolicySnapshot } {+ const billedCredits = Math.max(0, job.credits_cost ?? 0);+ if (!config.FEEDBACK_REFUND_ENABLED) return none(job, "refunds_disabled");+ if (billedCredits <= 0) return none(job, "zero_billed_credits");++ const policy = policyFor(job);+ if (!policy.ratings.includes(rating)) {+ return none(job, "rating_not_refundable", policy.ratings);+ }++ if (policy.mode === "flat") {+ return {+ desiredRefund: Math.min(policy.credits, billedCredits),+ policy: {+ version: "feedback_refund_v1",+ enabled: true,+ endpoint: job.endpoint,+ mode: "flat",+ refundableRatings: policy.ratings,+ matchedReason: policy.reason,+ flatCredits: policy.credits,+ maxCredits: policy.credits,+ },+ };+ }++ return {+ desiredRefund: Math.min(+ Math.ceil(billedCredits * policy.percent),+ policy.maxCredits,+ billedCredits,+ ),+ policy: {+ version: "feedback_refund_v1",+ enabled: true,+ endpoint: job.endpoint,+ mode: "percentage_with_cap",+ refundableRatings: policy.ratings,+ matchedReason: policy.reason,+ percent: policy.percent,+ maxCredits: policy.maxCredits,+ },+ };+}diff --git a/apps/api/src/controllers/v2/feedback/refund-totals.ts b/apps/api/src/controllers/v2/feedback/refund-totals.tsnew file mode 100644index 000000000..12a43249f--- /dev/null+++ b/apps/api/src/controllers/v2/feedback/refund-totals.ts@@ -0,0 +1,37 @@+import { and, eq, gte } from "drizzle-orm";+import { dbRr } from "../../../db/connection";+import * as schema from "../../../db/schema";+import { FeedbackLogger } from "./internal-types";++function startOfUtcDay(now: Date = new Date()): Date {+ const start = new Date(now.getTime());+ start.setUTCHours(0, 0, 0, 0);+ return start;+}++export async function sumCreditsRefundedToday(+ dbTeamId: string,+ logger: FeedbackLogger,+): Promise<number> {+ const since = startOfUtcDay().toISOString();++ try {+ const data = await dbRr+ .select({ credits_refunded: schema.search_feedback.credits_refunded })+ .from(schema.search_feedback)+ .where(+ and(+ eq(schema.search_feedback.team_id, dbTeamId),+ gte(schema.search_feedback.created_at, since),+ ),+ );++ return data.reduce((sum, row) => sum + (row.credits_refunded ?? 0), 0);+ } catch (error) {+ logger.warn(+ "Failed to compute feedback refund total; allowing refund this call",+ { error },+ );+ return 0;+ }+}diff --git a/apps/api/src/controllers/v2/feedback/request-input.ts b/apps/api/src/controllers/v2/feedback/request-input.tsnew file mode 100644index 000000000..d05d2d09a--- /dev/null+++ b/apps/api/src/controllers/v2/feedback/request-input.ts@@ -0,0 +1,27 @@+import { EndpointFeedbackRequest, SearchFeedbackRequest } from "../types";+import { FeedbackInput } from "./internal-types";++export function toFeedbackInput(+ body: EndpointFeedbackRequest | SearchFeedbackRequest,+): FeedbackInput {+ return {+ rating: body.rating,+ valuableSources: body.valuableSources,+ missingContent: body.missingContent,+ querySuggestions: body.querySuggestions,+ origin: body.origin,+ integration: body.integration,+ ...("issues" in body ? { issues: body.issues } : {}),+ ...("tags" in body ? { tags: body.tags } : {}),+ ...("note" in body ? { note: body.note } : {}),+ ...("url" in body ? { url: body.url } : {}),+ ...("pageNumbers" in body ? { pageNumbers: body.pageNumbers } : {}),+ ...("metadata" in body ? { metadata: body.metadata } : {}),+ };+}++export function toSearchFeedbackInput(+ body: SearchFeedbackRequest,+): FeedbackInput {+ return toFeedbackInput(body);+}diff --git a/apps/api/src/controllers/v2/feedback/zdr-persistence.test.ts b/apps/api/src/controllers/v2/feedback/zdr-persistence.test.tsnew file mode 100644index 000000000..6f7d8255e--- /dev/null+++ b/apps/api/src/controllers/v2/feedback/zdr-persistence.test.ts@@ -0,0 +1,179 @@+import {+ shouldSkipPersistenceForForcedZdr,+ shouldSkipPersistenceForJobZdr,+} from "./zdr-persistence";+import type { RequestWithAuth } from "../types";+import type { FeedbackJobRow, FeedbackRecordOptions } from "./internal-types";++function reqWithFlags(flags: Record<string, unknown>) {+ return {+ acuc: { flags },+ } as RequestWithAuth<any, any, any>;+}++function options(+ endpoint: FeedbackRecordOptions["endpoint"],+ overrides: Partial<FeedbackRecordOptions> = {},+): FeedbackRecordOptions {+ return {+ endpoint,+ jobId: "01933161-0000-7000-8000-000000000001",+ feedback: { rating: "bad", note: "bad result" },+ source: "feedback",+ ...overrides,+ };+}++function job(+ endpoint: FeedbackRecordOptions["endpoint"],+ jobOptions: unknown,+): FeedbackJobRow {+ return {+ endpoint,+ id: "01933161-0000-7000-8000-000000000001",+ request_id: "01933161-0000-7000-8000-000000000002",+ team_id: "01933161-0000-7000-8000-000000000003",+ credits_cost: 2,+ created_at: new Date().toISOString(),+ is_successful: true,+ options: jobOptions,+ };+}++describe("feedback ZDR persistence guards", () => {+ it("does not treat optional search ZDR as a forced persistence skip", () => {+ expect(+ shouldSkipPersistenceForForcedZdr(+ reqWithFlags({ searchZDR: "allowed" }),+ options("search"),+ ),+ ).toBe(false);+ });++ it("skips persistence for forced search ZDR modes", () => {+ expect(+ shouldSkipPersistenceForForcedZdr(+ reqWithFlags({ searchZDR: "forced-zdr" }),+ options("search"),+ ),+ ).toBe(true);++ expect(+ shouldSkipPersistenceForForcedZdr(+ reqWithFlags({ searchZDR: "forced-anon" }),+ options("search"),+ ),+ ).toBe(true);+ });++ it("does not treat optional scrape ZDR as a forced persistence skip", () => {+ expect(+ shouldSkipPersistenceForForcedZdr(+ reqWithFlags({ scrapeZDR: "allowed" }),+ options("scrape"),+ ),+ ).toBe(false);+ });++ it("does not treat legacy optional ZDR as a forced persistence skip", () => {+ expect(+ shouldSkipPersistenceForForcedZdr(+ reqWithFlags({ allowZDR: true }),+ options("search"),+ ),+ ).toBe(false);++ expect(+ shouldSkipPersistenceForForcedZdr(+ reqWithFlags({ allowZDR: true }),+ options("scrape"),+ ),+ ).toBe(false);+ });++ it("skips persistence for forced scrape and parse ZDR", () => {+ expect(+ shouldSkipPersistenceForForcedZdr(+ reqWithFlags({ scrapeZDR: "forced" }),+ options("scrape"),+ ),+ ).toBe(true);++ expect(+ shouldSkipPersistenceForForcedZdr(+ reqWithFlags({ scrapeZDR: "forced" }),+ options("parse"),+ ),+ ).toBe(true);+ });++ it("skips persistence for legacy forced ZDR", () => {+ expect(+ shouldSkipPersistenceForForcedZdr(+ reqWithFlags({ forceZDR: true }),+ options("search"),+ ),+ ).toBe(true);++ expect(+ shouldSkipPersistenceForForcedZdr(+ reqWithFlags({ forceZDR: true }),+ options("parse"),+ ),+ ).toBe(true);+ });++ it("does not skip map feedback for scrape ZDR flags", () => {+ expect(+ shouldSkipPersistenceForForcedZdr(+ reqWithFlags({ scrapeZDR: "forced" }),+ options("map"),+ ),+ ).toBe(false);+ });++ it("skips persistence for jobs logged with endpoint-level ZDR redaction", () => {+ expect(+ shouldSkipPersistenceForJobZdr(+ job("search", { enterprise: ["zdr"] }),+ options("search"),+ ),+ ).toBe(true);++ expect(+ shouldSkipPersistenceForJobZdr(job("scrape", null), options("scrape")),+ ).toBe(true);+ });++ it("allows persistence when the job row is not ZDR-redacted", () => {+ expect(+ shouldSkipPersistenceForJobZdr(+ job("search", { limit: 5 }),+ options("search"),+ ),+ ).toBe(false);++ expect(+ shouldSkipPersistenceForJobZdr(+ job("parse", { formats: ["markdown"] }),+ options("parse"),+ ),+ ).toBe(false);+ });++ it("honors explicit skipZdrPersistence override", () => {+ expect(+ shouldSkipPersistenceForForcedZdr(+ reqWithFlags({ searchZDR: "forced-zdr" }),+ options("search", { skipZdrPersistence: false }),+ ),+ ).toBe(false);++ expect(+ shouldSkipPersistenceForJobZdr(+ job("search", { enterprise: ["anon"] }),+ options("search", { skipZdrPersistence: false }),+ ),+ ).toBe(false);+ });+});diff --git a/apps/api/src/controllers/v2/feedback/zdr-persistence.ts b/apps/api/src/controllers/v2/feedback/zdr-persistence.tsnew file mode 100644index 000000000..47e6ddfd5--- /dev/null+++ b/apps/api/src/controllers/v2/feedback/zdr-persistence.ts@@ -0,0 +1,52 @@+import { getScrapeZDR, getSearchZDR } from "../../../lib/zdr-helpers";+import type { RequestWithAuth } from "../types";+import type { FeedbackJobRow, FeedbackRecordOptions } from "./internal-types";++type SearchOptions = {+ enterprise?: unknown;+};++function searchOptionsUseZdr(options: unknown): boolean {+ if (!options || typeof options !== "object") return false;++ const enterprise = (options as SearchOptions).enterprise;+ return (+ Array.isArray(enterprise) &&+ (enterprise.includes("zdr") || enterprise.includes("anon"))+ );+}++export function shouldSkipPersistenceForForcedZdr(+ req: RequestWithAuth<any, any, any>,+ options: FeedbackRecordOptions,+): boolean {+ if (options.skipZdrPersistence === false) return false;++ if (options.endpoint === "search") {+ const searchZDR = getSearchZDR(req.acuc?.flags);+ return searchZDR === "forced-zdr" || searchZDR === "forced-anon";+ }++ if (options.endpoint === "scrape" || options.endpoint === "parse") {+ return getScrapeZDR(req.acuc?.flags) === "forced";+ }++ return false;+}++export function shouldSkipPersistenceForJobZdr(+ job: FeedbackJobRow,+ options: FeedbackRecordOptions,+): boolean {+ if (options.skipZdrPersistence === false) return false;++ if (job.endpoint === "search") {+ return searchOptionsUseZdr(job.options);+ }++ if (job.endpoint === "scrape" || job.endpoint === "parse") {+ return job.options === null;+ }++ return false;+}diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.tsindex cb0803115..0728b662f 100644--- a/apps/api/src/controllers/v2/map.ts+++ b/apps/api/src/controllers/v2/map.ts@@ -117,6 +117,7 @@ export async function mapController(return res.status(200).json({success: true,+ id: mapId,links: avgrabResults,});}@@ -254,6 +255,7 @@ export async function mapController(const response = {success: true as const,+ id: result.job_id,links: result.mapResults,...(warning && { warning }),};diff --git a/apps/api/src/controllers/v2/search-feedback.ts b/apps/api/src/controllers/v2/search-feedback.tsindex c18c51cae..a6e7f4cab 100644--- a/apps/api/src/controllers/v2/search-feedback.ts+++ b/apps/api/src/controllers/v2/search-feedback.ts@@ -1,128 +1,15 @@import { Response } from "express";-import { v7 as uuidv7 } from "uuid";import { z } from "zod";-import { config } from "../../config";import { logger as _logger } from "../../lib/logger";-import {- autumnService,- SEARCH_CREDITS_FEATURE_ID,-} from "../../services/autumn/autumn.service";-import { and, eq, gte } from "drizzle-orm";-import { db, dbRr } from "../../db/connection";-import * as schema from "../../db/schema";-import { captureExceptionWithZdrCheck } from "../../services/sentry";-import { getScrapeZDR, getSearchZDR } from "../../lib/zdr-helpers";import {RequestWithAuth,- SearchFeedbackErrorCode,SearchFeedbackRequest,SearchFeedbackResponse,searchFeedbackSchema,} from "./types";--const SEARCH_FEEDBACK_REFUND_CREDITS = 1;--// Must match the previewTeamId in services/logging/log_job.ts.-const PREVIEW_TEAM_ID = "3adefd26-77ec-5968-8dcf-c94b5630d1de";--const POSTGRES_UNIQUE_VIOLATION = "23505";--// logSearch in the search controller is fire-and-forget; this retry covers-// the gap when feedback arrives before the searches row has been inserted.-const SEARCH_LOOKUP_RACE_RETRY_MS = 250;--type FailReason = {- status: number;- code: SearchFeedbackErrorCode;- error: string;-};--function fail(- res: Response<SearchFeedbackResponse>,- reason: FailReason,-): Response<SearchFeedbackResponse> {- return res.status(reason.status).json({- success: false,- error: reason.error,- feedbackErrorCode: reason.code,- });-}--function isPreviewTeam(teamId: string): boolean {- return teamId === "preview" || teamId.startsWith("preview_");-}--function normalizeTeamId(teamId: string): string {- return isPreviewTeam(teamId) ? PREVIEW_TEAM_ID : teamId;-}--type SearchRowForFeedback = {- id: string;- team_id: string;- credits_cost: number | null;- created_at: string;- is_successful: boolean | null;-};--async function lookupSearchRow(- searchId: string,- dbTeamId: string,-): Promise<SearchRowForFeedback | null> {- const [data] = await dbRr- .select({- id: schema.searches.id,- team_id: schema.searches.team_id,- credits_cost: schema.searches.credits_cost,- created_at: schema.searches.created_at,- is_successful: schema.searches.is_successful,- })- .from(schema.searches)- .where(- and(- eq(schema.searches.id, searchId),- eq(schema.searches.team_id, dbTeamId),- ),- )- .limit(1);-- return (data ?? null) as SearchRowForFeedback | null;-}--function startOfUtcDay(now: Date = new Date()): Date {- const start = new Date(now.getTime());- start.setUTCHours(0, 0, 0, 0);- return start;-}--// Best-effort: on DB error we return 0 rather than block legitimate-// refunds; overshoot is bounded by the rate limiter.-async function sumTeamCreditsRefundedToday(- dbTeamId: string,- logger: ReturnType<typeof _logger.child>,-): Promise<number> {- const since = startOfUtcDay().toISOString();-- let data: { credits_refunded: number | null }[];- try {- data = await dbRr- .select({ credits_refunded: schema.search_feedback.credits_refunded })- .from(schema.search_feedback)- .where(- and(- eq(schema.search_feedback.team_id, dbTeamId),- gte(schema.search_feedback.created_at, since),- ),- );- } catch (error) {- logger.warn(- "Failed to compute today's refund total; allowing refund this call",- { error },- );- return 0;- }-- return data.reduce((sum, row) => sum + (row.credits_refunded ?? 0), 0);-}+import { recordEndpointFeedback } from "./feedback/record";+import { searchFeedbackRecordOptions } from "./feedback/record-options";+import { toSearchFeedbackInput } from "./feedback/request-input";export async function searchFeedbackController(req: RequestWithAuth<@@ -156,266 +43,13 @@ export async function searchFeedbackController(throw error;}- // ZDR teams must not have any feedback persisted. Short-circuit with a- // success-shaped response so clients behave normally, but record nothing- // and refund nothing.- const searchZDR = getSearchZDR(req.acuc?.flags);- const isZDR =- getScrapeZDR(req.acuc?.flags) !== "disabled" ||- searchZDR === "allowed" ||- searchZDR === "forced-zdr" ||- searchZDR === "forced-anon";- if (isZDR) {- return res.status(200).json({- success: true,- feedbackId: "00000000-0000-0000-0000-000000000000",- creditsRefunded: 0,- creditsRefundedToday: 0,- dailyRefundCap: config.SEARCH_FEEDBACK_DAILY_CAP_CREDITS,- });- }-- if (config.USE_DB_AUTHENTICATION !== true) {- return fail(res, {- status: 503,- code: "DB_DISABLED",- error:- "Search feedback requires database authentication and is unavailable on this deployment.",- });- }-- // Preview tenants share one team_id in the searches table, so one- // preview agent could otherwise submit feedback on another's searches.- if (isPreviewTeam(req.auth.team_id)) {- return fail(res, {- status: 403,- code: "PREVIEW_TEAM_NOT_ALLOWED",- error: "Search feedback is not available for preview teams.",- });- }-- if (req.acuc?.flags?.searchFeedbackOptOut === true) {- logger.info("Rejected feedback: team opted out");- return fail(res, {- status: 403,- code: "TEAM_OPTED_OUT",- error:- "Search feedback is disabled for this team. Contact [email protected] to re-enable.",- });- }+ const result = await recordEndpointFeedback(+ req,+ searchFeedbackRecordOptions({+ jobId: searchId,+ feedback: toSearchFeedbackInput(parsedBody),+ }),+ );- const dbTeamId = normalizeTeamId(req.auth.team_id);-- try {- let searchRow: SearchRowForFeedback | null;- try {- searchRow = await lookupSearchRow(searchId, dbTeamId);- if (!searchRow) {- await new Promise(resolve =>- setTimeout(resolve, SEARCH_LOOKUP_RACE_RETRY_MS),- );- searchRow = await lookupSearchRow(searchId, dbTeamId);- }- } catch (lookupErr) {- logger.error("Failed to look up search for feedback", {- error: lookupErr,- });- return fail(res, {- status: 500,- code: "INTERNAL",- error: "Failed to look up search.",- });- }-- if (!searchRow) {- return fail(res, {- status: 404,- code: "SEARCH_NOT_FOUND",- error: "Search not found for this team.",- });- }-- if (searchRow.is_successful === false) {- return fail(res, {- status: 409,- code: "SEARCH_FAILED",- error: "Cannot submit feedback for a search that did not succeed.",- });- }-- const maxAgeMs = config.SEARCH_FEEDBACK_MAX_AGE_SEC * 1000;- const createdAtMs = new Date(searchRow.created_at).getTime();- if (Number.isNaN(createdAtMs)) {- logger.warn("Search row had unparseable created_at", {- created_at: searchRow.created_at,- });- } else {- const ageMs = Date.now() - createdAtMs;- if (ageMs > maxAgeMs) {- logger.info("Rejected feedback outside time window", {- ageMs,- maxAgeMs,- });- return fail(res, {- status: 409,- code: "FEEDBACK_WINDOW_EXPIRED",- error: `Search feedback must be submitted within ${config.SEARCH_FEEDBACK_MAX_AGE_SEC} seconds of the search.`,- });- }- }-- const feedbackId = uuidv7();- try {- await db.insert(schema.search_feedback).values({- id: feedbackId,- search_id: searchId,- team_id: dbTeamId,- overall_rating: parsedBody.rating,- valuable_sources: parsedBody.valuableSources ?? [],- missing_content: parsedBody.missingContent ?? [],- query_suggestions: parsedBody.querySuggestions ?? null,- integration: parsedBody.integration ?? null,- origin: parsedBody.origin ?? null,- credits_refunded: 0,- });- } catch (insertErr) {- if ((insertErr as { code?: string }).code === POSTGRES_UNIQUE_VIOLATION) {- const [existing] = await dbRr- .select({- id: schema.search_feedback.id,- credits_refunded: schema.search_feedback.credits_refunded,- })- .from(schema.search_feedback)- .where(eq(schema.search_feedback.search_id, searchId))- .limit(1);-- const dailyCapForResponse = config.SEARCH_FEEDBACK_DAILY_CAP_CREDITS;- const refundedTodayForResponse = await sumTeamCreditsRefundedToday(- dbTeamId,- logger,- );-- return res.status(200).json({- success: true,- feedbackId: existing?.id ?? "",- creditsRefunded: 0,- alreadySubmitted: true,- creditsRefundedToday: refundedTodayForResponse,- dailyRefundCap: dailyCapForResponse,- warning:- "Feedback was already submitted for this search; no additional refund issued.",- });- }-- logger.error("Failed to insert search feedback", { error: insertErr });- return fail(res, {- status: 500,- code: "INTERNAL",- error: "Failed to record feedback.",- });- }-- // The just-inserted row has credits_refunded=0, so reading the SUM- // here is safe whether or not it shows up in the read replica yet.- const dailyCap = config.SEARCH_FEEDBACK_DAILY_CAP_CREDITS;- const refundedTodayBefore = await sumTeamCreditsRefundedToday(- dbTeamId,- logger,- );- const remainingDailyCap = Math.max(0, dailyCap - refundedTodayBefore);-- let creditsRefunded = 0;- let dailyCapReached = false;- const billedCredits = searchRow.credits_cost ?? 0;-- const desiredRefund = Math.min(- SEARCH_FEEDBACK_REFUND_CREDITS,- billedCredits,- );- const cappedRefund = Math.min(desiredRefund, remainingDailyCap);-- if (billedCredits > 0 && desiredRefund > 0 && cappedRefund === 0) {- dailyCapReached = true;- logger.info(- "Daily refund cap reached for team; feedback recorded with zero refund",- {- dailyCap,- refundedTodayBefore,- },- );- } else if (cappedRefund > 0) {- try {- await autumnService.refundCredits({- teamId: req.auth.team_id,- value: cappedRefund,- properties: {- source: "search_feedback",- endpoint: "search",- jobId: searchId,- feedbackId,- rating: parsedBody.rating,- },- featureId: SEARCH_CREDITS_FEATURE_ID,- });- creditsRefunded = cappedRefund;- } catch (error) {- logger.error("Search feedback refund failed; feedback retained", {- error,- });- }-- if (creditsRefunded > 0) {- try {- await db- .update(schema.search_feedback)- .set({ credits_refunded: creditsRefunded })- .where(eq(schema.search_feedback.id, feedbackId));- } catch (updateErr) {- logger.warn(- "Failed to persist credits_refunded on search_feedback row",- { error: updateErr, feedbackId, creditsRefunded },- );- }- }- }-- const creditsRefundedToday = refundedTodayBefore + creditsRefunded;- if (!dailyCapReached && creditsRefundedToday >= dailyCap && dailyCap > 0) {- dailyCapReached = true;- }-- logger.info("Search feedback recorded", {- feedbackId,- creditsRefunded,- rating: parsedBody.rating,- valuableSourcesCount: parsedBody.valuableSources?.length ?? 0,- missingContentCount: parsedBody.missingContent?.length ?? 0,- hasQuerySuggestions: !!parsedBody.querySuggestions,- creditsRefundedToday,- dailyRefundCap: dailyCap,- dailyCapReached,- });-- return res.status(200).json({- success: true,- feedbackId,- creditsRefunded,- creditsRefundedToday,- dailyRefundCap: dailyCap,- ...(dailyCapReached- ? {- dailyCapReached: true,- warning: `Daily refund cap of ${dailyCap} credits reached for this team (UTC day). Feedback was recorded; further /feedback calls today will not refund credits.`,- }- : {}),- });- } catch (error) {- captureExceptionWithZdrCheck(error);- logger.error("Unhandled error in search feedback controller", { error });- return fail(res, {- status: 500,- code: "INTERNAL",- error: error instanceof Error ? error.message : "Unknown error",- });- }+ return res.status(result.status).json(result.body);}diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 764d7020a..1f9e3e9ef 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1424,6 +1424,7 @@ export type MapResponse =| ErrorResponse| {success: true;+ id: string;links?: MapDocument[];warning?: string;};@@ -2098,6 +2099,29 @@ const missingContentEntrySchema = z.strictObject({description: z.string().trim().max(2000).optional(),});+function hasSubstantiveSearchFeedback(data: {+ rating: "good" | "bad" | "partial";+ valuableSources?: unknown[];+ missingContent?: unknown[];+ querySuggestions?: string;+}): boolean {+ const hasSources = (data.valuableSources?.length ?? 0) > 0;+ const hasMissing = (data.missingContent?.length ?? 0) > 0;+ const hasSuggestions =+ !!data.querySuggestions && data.querySuggestions.length > 0;++ switch (data.rating) {+ case "good":+ return hasSources;+ case "partial":+ return hasSources || hasMissing;+ case "bad":+ return hasMissing || hasSuggestions;+ }++ return false;+}+export const searchFeedbackSchema = z.strictObject({rating: z.enum(["good", "bad", "partial"]),@@ -2115,27 +2139,10 @@ export const searchFeedbackSchema = zorigin: z.string().optional().prefault("api"),integration: integrationSchema.optional().transform(val => val || null),})- .refine(- data => {- const hasSources = (data.valuableSources?.length ?? 0) > 0;- const hasMissing = (data.missingContent?.length ?? 0) > 0;- const hasSuggestions =- !!data.querySuggestions && data.querySuggestions.length > 0;-- switch (data.rating) {- case "good":- return hasSources;- case "partial":- return hasSources || hasMissing;- case "bad":- return hasMissing || hasSuggestions;- }- },- {- message:- "Feedback must be substantive. 'good' requires at least one valuableSources entry; 'partial' requires valuableSources or at least one missingContent entry; 'bad' requires at least one missingContent entry or querySuggestions.",- },- );+ .refine(data => hasSubstantiveSearchFeedback(data), {+ message:+ "Feedback must be substantive. 'good' requires at least one valuableSources entry; 'partial' requires valuableSources or at least one missingContent entry; 'bad' requires at least one missingContent entry or querySuggestions.",+ });export type SearchFeedbackRequest = z.infer<typeof searchFeedbackSchema>;export type SearchFeedbackRequestInput = z.input<typeof searchFeedbackSchema>;@@ -2163,6 +2170,119 @@ export type SearchFeedbackResponse =warning?: string;};+// =============================================+// Generic Endpoint Feedback+// =============================================++const endpointFeedbackEndpointSchema = z.enum([+ "search",+ "scrape",+ "parse",+ "map",+]);++export type EndpointFeedbackEndpoint = z.infer<+ typeof endpointFeedbackEndpointSchema+>;++const feedbackIssueSchema = z+ .string()+ .trim()+ .min(1)+ .max(80)+ .regex(+ /^[a-z0-9][a-z0-9_-]*$/,+ "Issue codes must use lowercase letters, numbers, underscores, or hyphens",+ );++const MAX_FEEDBACK_METADATA_BYTES = 8 * 1024;+const feedbackMetadataSchema = z+ .record(z.string(), z.unknown())+ .refine(+ value =>+ new TextEncoder().encode(JSON.stringify(value)).length <=+ MAX_FEEDBACK_METADATA_BYTES,+ "metadata must be 8KB or smaller",+ );++export const endpointFeedbackSchema = z+ .strictObject({+ endpoint: endpointFeedbackEndpointSchema,+ jobId: z.uuid(),+ rating: z.enum(["good", "bad", "partial"]),+ issues: z.array(feedbackIssueSchema).max(20).optional(),+ tags: z.array(feedbackIssueSchema).max(20).optional(),+ note: z.string().trim().max(4000).optional(),+ valuableSources: z+ .array(+ z.strictObject({+ url: searchFeedbackUrlSchema,+ reason: z.string().trim().max(1000).optional(),+ }),+ )+ .max(50)+ .optional(),+ missingContent: z.array(missingContentEntrySchema).max(50).optional(),+ querySuggestions: z.string().trim().max(2000).optional(),+ url: searchFeedbackUrlSchema.optional(),+ pageNumbers: z.array(z.number().int().positive()).max(100).optional(),+ metadata: feedbackMetadataSchema.optional(),+ origin: z.string().optional().prefault("api"),+ integration: integrationSchema.optional().transform(val => val || null),+ })+ .refine(+ data => {+ return (+ (data.issues?.length ?? 0) > 0 ||+ (data.tags?.length ?? 0) > 0 ||+ (data.note?.length ?? 0) > 0 ||+ (data.valuableSources?.length ?? 0) > 0 ||+ (data.missingContent?.length ?? 0) > 0 ||+ !!data.querySuggestions ||+ !!data.url ||+ (data.pageNumbers?.length ?? 0) > 0+ );+ },+ {+ message:+ "Feedback must include at least one substantive signal: issues, note, sources, missingContent, querySuggestions, url, or pageNumbers.",+ },+ )+ .refine(+ data => data.endpoint !== "search" || hasSubstantiveSearchFeedback(data),+ {+ message:+ "Search feedback must be substantive. 'good' requires at least one valuableSources entry; 'partial' requires valuableSources or at least one missingContent entry; 'bad' requires at least one missingContent entry or querySuggestions.",+ },+ );++export type EndpointFeedbackRequest = z.infer<typeof endpointFeedbackSchema>;+export type EndpointFeedbackRequestInput = z.input<+ typeof endpointFeedbackSchema+>;++export type EndpointFeedbackErrorCode =+ | "JOB_NOT_FOUND"+ | "FEEDBACK_WINDOW_EXPIRED"+ | "PREVIEW_TEAM_NOT_ALLOWED"+ | "TEAM_OPTED_OUT"+ | "INVALID_BODY"+ | "DB_DISABLED"+ | "INTERNAL";++export type EndpointFeedbackResponse =+ | (ErrorResponse & { feedbackErrorCode?: EndpointFeedbackErrorCode })+ | {+ success: true;+ feedbackId: string;+ creditsRefunded: number;+ alreadySubmitted?: boolean;+ dailyCapReached?: boolean;+ creditsRefundedToday?: number;+ dailyRefundCap?: number;+ warning?: string;+ };+export type TokenUsage = {promptTokens: number;completionTokens: number;diff --git a/apps/api/src/db/schema/public.ts b/apps/api/src/db/schema/public.tsindex e5d46af89..f73752a35 100644--- a/apps/api/src/db/schema/public.ts+++ b/apps/api/src/db/schema/public.ts@@ -467,16 +467,29 @@ export const scrapes = pgTable("scrapes", {export const search_feedback = pgTable("search_feedback", {id: uuid("id").notNull().defaultRandom(),- search_id: uuid("search_id").notNull(),+ search_id: uuid("search_id"),+ endpoint: text("endpoint").notNull().default("search"),+ job_id: uuid("job_id"),+ request_id: uuid("request_id"),+ api_version: text("api_version").default("v2"),team_id: uuid("team_id").notNull(),+ api_key_id: bigintNum("api_key_id"),overall_rating: text("overall_rating").notNull(),+ issue_types: text("issue_types").array().notNull().default([]),+ tags: text("tags").array().notNull().default([]),+ comment: text("comment"),valuable_sources: jsonb("valuable_sources").notNull().default([]),missing_content: jsonb("missing_content").notNull().default([]),query_suggestions: text("query_suggestions"),+ metadata: jsonb("metadata").notNull().default({}),+ job_status: text("job_status"),+ credits_billed: integer("credits_billed").notNull().default(0),integration: text("integration"),origin: text("origin"),credits_refunded: integer("credits_refunded").notNull().default(0),+ refund_policy: jsonb("refund_policy"),created_at: ts("created_at").notNull().defaultNow(),+ updated_at: ts("updated_at").notNull().defaultNow(),});export const searches = pgTable("searches", {diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.tsindex 0561b0b7d..a7cd75821 100644--- a/apps/api/src/routes/v2.ts+++ b/apps/api/src/routes/v2.ts@@ -5,6 +5,7 @@ import { RateLimiterMode } from "../types";import { SEARCH_CREDITS_FEATURE_ID } from "../services/autumn/autumn.service";import expressWs from "express-ws";import { searchController } from "../controllers/v2/search";+import { feedbackController } from "../controllers/v2/feedback/controller";import { searchFeedbackController } from "../controllers/v2/search-feedback";import { x402SearchController } from "../controllers/v2/x402-search";import { scrapeController } from "../controllers/v2/scrape";@@ -255,6 +256,12 @@ v2Router.post(wrap(searchFeedbackController),);+v2Router.post(+ "/feedback",+ authMiddleware(RateLimiterMode.Account),+ wrap(feedbackController),+);+v2Router.post("/parse",authMiddleware(RateLimiterMode.Scrape),diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex 0f607d1f5..325a35d48 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -273,9 +273,7 @@ export async function scrapeURLWithIndex(const value =!data || data.length === 0 ? null : (data[0].max_age ?? null);if (useIndexCache) {- setCachedMaxAge(domainHash, value, meta.logger).catch(- () => {},- );+ setCachedMaxAge(domainHash, value, meta.logger).catch(() => {});}return { value: value ?? defaultMaxAge, source: "dynamic_db" };} catch (error) {@@ -283,11 +281,10 @@ export async function scrapeURLWithIndex(return { value: defaultMaxAge, source: "default" };}})(),- new Promise<{ value: number; source: MaxAgeSource }>(- resolve =>- setTimeout(() => {- resolve({ value: defaultMaxAge, source: "default" });- }, 200),+ new Promise<{ value: number; source: MaxAgeSource }>(resolve =>+ setTimeout(() => {+ resolve({ value: defaultMaxAge, source: "default" });+ }, 200),),]);maxAge = resolved.value;@@ -404,7 +401,10 @@ export async function scrapeURLWithIndex(const negStart = Date.now();const neg = await getCachedNegative(variantKey, meta.logger);timingsCache += Date.now() - negStart;- if (neg !== null && isNegativeStillValid(neg.emptyFrom, maxAge, Date.now())) {+ if (+ neg !== null &&+ isNegativeStillValid(neg.emptyFrom, maxAge, Date.now())+ ) {negativeHit = true;}}diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.tsindex c252c467b..be84f45c7 100644--- a/apps/api/src/search/execute.ts+++ b/apps/api/src/search/execute.ts@@ -194,7 +194,11 @@ export async function executeSearch(// Runs after scraping (mergeScrapedContent rebuilds the result objects, so// highlight mutations must come last to survive). Uses the user's original// query, not the domain-filtered upstream query.- if (options.highlights && flags?.highlightsBeta === true && highlightsEnvReady()) {+ if (+ options.highlights &&+ flags?.highlightsBeta === true &&+ highlightsEnvReady()+ ) {await applySearchHighlights(searchResponse, query, logger);}diff --git a/apps/api/src/search/highlights.ts b/apps/api/src/search/highlights.tsindex 116f5ff0d..4bb4f44c3 100644--- a/apps/api/src/search/highlights.ts+++ b/apps/api/src/search/highlights.ts@@ -156,9 +156,15 @@ export async function applySearchHighlights(for (const result of response.web ?? []) {if (!result.url) continue;tasks.push(- highlightOne(result.url, query, logger, h => {- result.description = h;- }, counters),+ highlightOne(+ result.url,+ query,+ logger,+ h => {+ result.description = h;+ },+ counters,+ ),);}@@ -167,9 +173,15 @@ export async function applySearchHighlights(if (!result.url) continue;const url = result.url;tasks.push(- highlightOne(url, query, logger, h => {- result.snippet = h;- }, counters),+ highlightOne(+ url,+ query,+ logger,+ h => {+ result.snippet = h;+ },+ counters,+ ),);}@@ -183,5 +195,9 @@ export async function applySearchHighlights(timeTakenMs: Date.now() - start,});- return { attempted, indexHits: counters.indexHits, replaced: counters.replaced };+ return {+ attempted,+ indexHits: counters.indexHits,+ replaced: counters.replaced,+ };}diff --git a/apps/api/src/services/worker/nuq-worker.ts b/apps/api/src/services/worker/nuq-worker.tsindex 984ba3f9d..dab1132f8 100644--- a/apps/api/src/services/worker/nuq-worker.ts+++ b/apps/api/src/services/worker/nuq-worker.ts@@ -1,11 +1,7 @@import "dotenv/config";import "../sentry";import { setSentryServiceTag } from "../sentry";-import {- nuqGetLocalMetrics,- nuqHealthCheck,- scrapeQueue,-} from "./nuq";+import { nuqGetLocalMetrics, nuqHealthCheck, scrapeQueue } from "./nuq";import { runNuqWorker } from "./nuq-worker-runner";(async () => {diff --git a/apps/js-sdk/firecrawl/src/v2/client.ts b/apps/js-sdk/firecrawl/src/v2/client.tsindex bd75c7285..42273692b 100644--- a/apps/js-sdk/firecrawl/src/v2/client.ts+++ b/apps/js-sdk/firecrawl/src/v2/client.ts@@ -7,6 +7,7 @@ import {import { parse as parseMethod } from "./methods/parse";import { search } from "./methods/search";import { map as mapMethod } from "./methods/map";+import { feedback as feedbackMethod, searchFeedback as searchFeedbackMethod } from "./methods/feedback";import {startCrawl,getCrawlStatus,@@ -50,6 +51,9 @@ import type {ScrapeOptions,SearchData,SearchRequest,+ EndpointFeedbackRequest,+ FeedbackResponse,+ SearchFeedbackRequest,MapData,MapOptions,CrawlResponse,@@ -236,6 +240,25 @@ export class FirecrawlClient {return search(this.http, { query, ...req });}+ /**+ * Submit feedback for a v2 job.+ * @param request Feedback payload with endpoint, job id, rating, and supporting signals.+ * @returns Feedback record and refund details.+ */+ async feedback(request: EndpointFeedbackRequest): Promise<FeedbackResponse> {+ return feedbackMethod(this.http, request);+ }++ /**+ * Submit feedback for a search job.+ * @param jobId Search job id returned by search.+ * @param request Search feedback payload.+ * @returns Feedback record and refund details.+ */+ async searchFeedback(jobId: string, request: SearchFeedbackRequest): Promise<FeedbackResponse> {+ return searchFeedbackMethod(this.http, jobId, request);+ }+// Research/*** Access the v2 research endpoints (arXiv papers + GitHub history/readmes).@@ -618,4 +641,3 @@ export class FirecrawlClient {}export default FirecrawlClient;-diff --git a/apps/js-sdk/firecrawl/src/v2/methods/feedback.ts b/apps/js-sdk/firecrawl/src/v2/methods/feedback.tsnew file mode 100644index 000000000..7af68a10a--- /dev/null+++ b/apps/js-sdk/firecrawl/src/v2/methods/feedback.ts@@ -0,0 +1,59 @@+import {+ type EndpointFeedbackRequest,+ type FeedbackResponse,+ type SearchFeedbackRequest,+} from "../types";+import { HttpClient } from "../utils/httpClient";+import {+ normalizeAxiosError,+ throwForBadResponse,+} from "../utils/errorHandler";++function validateRating(rating: string): void {+ if (!["good", "partial", "bad"].includes(rating)) {+ throw new Error("rating must be one of: good, partial, bad");+ }+}++export async function feedback(+ http: HttpClient,+ request: EndpointFeedbackRequest,+): Promise<FeedbackResponse> {+ if (!request.endpoint) throw new Error("endpoint is required");+ if (!request.jobId) throw new Error("jobId is required");+ validateRating(request.rating);++ try {+ const res = await http.post<FeedbackResponse>("/v2/feedback", request);+ if (res.status !== 200 || !res.data?.success) {+ throwForBadResponse(res, "feedback");+ }+ return res.data;+ } catch (err: any) {+ if (err?.isAxiosError) return normalizeAxiosError(err, "feedback");+ throw err;+ }+}++export async function searchFeedback(+ http: HttpClient,+ jobId: string,+ request: SearchFeedbackRequest,+): Promise<FeedbackResponse> {+ if (!jobId) throw new Error("jobId is required");+ validateRating(request.rating);++ try {+ const res = await http.post<FeedbackResponse>(+ `/v2/search/${encodeURIComponent(jobId)}/feedback`,+ request,+ );+ if (res.status !== 200 || !res.data?.success) {+ throwForBadResponse(res, "searchFeedback");+ }+ return res.data;+ } catch (err: any) {+ if (err?.isAxiosError) return normalizeAxiosError(err, "searchFeedback");+ throw err;+ }+}diff --git a/apps/js-sdk/firecrawl/src/v2/methods/map.ts b/apps/js-sdk/firecrawl/src/v2/methods/map.tsindex 7958fb361..5a887ca82 100644--- a/apps/js-sdk/firecrawl/src/v2/methods/map.ts+++ b/apps/js-sdk/firecrawl/src/v2/methods/map.ts@@ -37,6 +37,7 @@ export async function map(try {const res = await http.post<{success: boolean;+ id?: string;error?: string;links?: Array<string | SearchResultWeb>;}>(@@ -60,7 +61,7 @@ export async function map(description: (item as any).description,});}- return { links };+ return { id: res.data.id, links };} catch (err: any) {if (err?.isAxiosError) return normalizeAxiosError(err, "map");throw err;diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex c6e5e4408..edc7a8378 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -647,6 +647,7 @@ export interface BatchScrapeJob {}export interface MapData {+ id?: string;links: SearchResultWeb[];}@@ -662,6 +663,51 @@ export interface MapOptions {location?: LocationConfig;}+export type FeedbackRating = "good" | "partial" | "bad";+export type EndpointFeedbackEndpoint = "search" | "scrape" | "parse" | "map";++export interface FeedbackValuableSource {+ url: string;+ reason?: string;+}++export interface FeedbackMissingContent {+ topic: string;+ description?: string;+}++export interface SearchFeedbackRequest {+ rating: FeedbackRating;+ valuableSources?: FeedbackValuableSource[];+ missingContent?: FeedbackMissingContent[];+ querySuggestions?: string;+ integration?: string | null;+ origin?: string;+}++export interface EndpointFeedbackRequest extends SearchFeedbackRequest {+ endpoint: EndpointFeedbackEndpoint;+ jobId: string;+ issues?: string[];+ tags?: string[];+ note?: string;+ url?: string;+ pageNumbers?: number[];+ /** Small endpoint-specific metadata object. Must be 8KB or smaller. */+ metadata?: Record<string, unknown>;+}++export interface FeedbackResponse {+ success: true;+ feedbackId: string;+ creditsRefunded: number;+ alreadySubmitted?: boolean;+ dailyCapReached?: boolean;+ creditsRefundedToday?: number;+ dailyRefundCap?: number;+ warning?: string;+}+/*** Schedule for a monitor.*