Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): check Redis pipeline results instead of swallowing command errors (#4557)
Gergő Móricze4ac89b02568
diff --git a/apps/api/src/__tests__/snips/index-cache.test.ts b/apps/api/src/__tests__/snips/index-cache.test.tsindex 9d4366e41..ecf11dfdb 100644--- a/apps/api/src/__tests__/snips/index-cache.test.ts+++ b/apps/api/src/__tests__/snips/index-cache.test.ts@@ -273,3 +273,68 @@ describe("index cache fail-open", () => {).resolves.toBeUndefined();}, 5000);});++describe("upsertCachedIndexEntries pipeline error handling", () => {+ function fakeClientWithResults(results: [Error | null, unknown][]) {+ const delCalls: string[] = [];+ const client: any = {+ pipeline: () => {+ const pipeline: any = {+ hset: () => pipeline,+ expire: () => pipeline,+ del: () => pipeline,+ hlen: () => pipeline,+ exec: async () => results,+ };+ return pipeline;+ },+ del: async (key: string) => {+ delCalls.push(key);+ return 1;+ },+ };+ return { client, delCalls };+ }++ it("retries the negative-marker DEL when the HSET landed but the DEL failed", async () => {+ const { client, delCalls } = fakeClientWithResults([+ [null, 1], // hset+ [null, 1], // expire+ [new Error("boom"), undefined], // del negative marker+ [null, 1], // hlen+ ]);++ await upsertCachedIndexEntries("idxc:test", [entry()], undefined, client);++ expect(delCalls).toEqual(["idxcneg:test"]);+ });++ it("still retries the negative-marker DEL when the HSET failed", async () => {+ // A connection failure can report the HSET as failed even when Redis+ // applied it, and deleting the marker is always safe (it only costs a+ // DB read on the next lookup).+ const { client, delCalls } = fakeClientWithResults([+ [new Error("boom"), undefined], // hset+ [null, 1], // expire+ [new Error("boom"), undefined], // del negative marker+ [null, 1], // hlen+ ]);++ await upsertCachedIndexEntries("idxc:test", [entry()], undefined, client);++ expect(delCalls).toEqual(["idxcneg:test"]);+ });++ it("does not retry the negative-marker DEL when it succeeded", async () => {+ const { client, delCalls } = fakeClientWithResults([+ [null, 1], // hset+ [null, 1], // expire+ [null, 1], // del negative marker+ [null, 1], // hlen+ ]);++ await upsertCachedIndexEntries("idxc:test", [entry()], undefined, client);++ expect(delCalls).toEqual([]);+ });+});diff --git a/apps/api/src/controllers/v2/parse-upload.ts b/apps/api/src/controllers/v2/parse-upload.tsindex e4f1acde6..d3f204ada 100644--- a/apps/api/src/controllers/v2/parse-upload.ts+++ b/apps/api/src/controllers/v2/parse-upload.ts@@ -11,6 +11,7 @@ import { getRedisConnection } from "../../services/queue-service";import { RequestWithAuth, UploadedParseFile } from "./types";import { detectUploadedFileKind, getSupportedParseFileTypes } from "./parse";import { isImageOcrEnabled } from "../../lib/image-ocr-gate";+import { reportPipelineError } from "../../lib/redis-pipeline";const PARSE_UPLOAD_MAX_BYTES = 50 * 1024 * 1024;const PARSE_UPLOAD_TTL_MS = 10 * 60 * 1000;@@ -226,12 +227,25 @@ async function reserveUnparsedUploadRef(teamId: string, uploadId: string) {async function releaseUnparsedUploadRef(teamId: string, uploadId: string) {const cutoff = Date.now() - PARSE_UPLOAD_UNPARSED_WINDOW_MS;const key = getUnparsedUploadsKey(teamId);- await getRedisConnection()+ const results = await getRedisConnection().pipeline().zremrangebyscore(key, "-inf", cutoff).zrem(key, uploadId).expire(key, PARSE_UPLOAD_UNPARSED_TTL_SECONDS).exec();+ // Both call sites catch and log a throw here; a leaked ref self-expires+ // via TTL.+ const error = reportPipelineError(results, _logger, {+ module: "parse-upload",+ method: "releaseUnparsedUploadRef",+ teamId,+ uploadId,+ });+ if (error) {+ throw new Error("Failed to release unparsed upload ref: " + error.message, {+ cause: error,+ });+ }}async function reserveUnparsedUploadRefOrRespond(diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.tsindex 246e376f0..db98cd744 100644--- a/apps/api/src/lib/concurrency-limit.ts+++ b/apps/api/src/lib/concurrency-limit.ts@@ -19,6 +19,7 @@ import {removeConcurrencyLimitActiveJob,} from "./concurrency-redis";import { autumnService } from "../services/autumn/autumn.service";+import { reportPipelineError } from "./redis-pipeline";// Fallback when Autumn can't give us a concurrency value.const DEFAULT_CONCURRENCY_LIMIT = 2;@@ -83,7 +84,15 @@ export async function removeConcurrencyLimitedJobs(for (const id of chunk) {pipeline.del(constructJobKey(id));}- await pipeline.exec();+ // Do not throw on command errors: cancel has already been recorded on+ // the crawl, and the stale entries self-expire via their PX timeout.+ // But never let the failure pass silently.+ reportPipelineError(await pipeline.exec(), logger, {+ module: "concurrency-limit",+ method: "removeConcurrencyLimitedJobs",+ teamId: team_id,+ jobCount: chunk.length,+ });}}@@ -143,7 +152,16 @@ export async function pushConcurrencyLimitedJobs(pipeline.zadd(queueKey, ...zaddArgs);pipeline.sadd("concurrency-limit-queues", queueKey);- await pipeline.exec();+ // Do not throw on command errors: the jobs are already durable in the+ // NuQ backlog, and the concurrency-queue reconciler requeues anything+ // missing from this derived Redis index on its next run. But never let+ // the failure pass silently.+ reportPipelineError(await pipeline.exec(), logger, {+ module: "concurrency-limit",+ method: "pushConcurrencyLimitedJobs",+ teamId: team_id,+ jobCount: jobs.length,+ });}export async function getConcurrencyLimitedJobs(team_id: string) {diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex eb9d8cb7a..f9a2b1ee1 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -8,6 +8,11 @@ import { getAdjustedMaxDepth } from "../scraper/WebScraper/utils/maxDepthUtils";import type { Logger } from "winston";import { withSpan, setSpanAttributes } from "./otel-tracer";import { getScrapeZDR, getIgnoreRobots } from "./zdr-helpers";+import {+ firstPipelineError,+ reportPipelineError,+ REDIS_COMMAND_ARG_CHUNK_SIZE,+} from "./redis-pipeline";export type StoredCrawl = {originUrl?: string;@@ -167,7 +172,17 @@ export async function addCrawlJob(pipeline.expire("crawl:" + id + ":jobs", 24 * 60 * 60);pipeline.sadd("crawl:" + id + ":jobs_qualified", job_id);pipeline.expire("crawl:" + id + ":jobs_qualified", 24 * 60 * 60);- await pipeline.exec();+ const error = reportPipelineError(await pipeline.exec(), __logger, {+ module: "crawl-redis",+ method: "addCrawlJob",+ crawlId: id,+ jobId: job_id,+ });+ if (error) {+ throw new Error("Failed to add crawl job: " + error.message, {+ cause: error,+ });+ }});}@@ -185,11 +200,24 @@ export async function addCrawlJobs(crawlId: id,});const pipeline = redisEvictConnection.pipeline();- pipeline.sadd("crawl:" + id + ":jobs", ...job_ids);+ for (let i = 0; i < job_ids.length; i += REDIS_COMMAND_ARG_CHUNK_SIZE) {+ const chunk = job_ids.slice(i, i + REDIS_COMMAND_ARG_CHUNK_SIZE);+ pipeline.sadd("crawl:" + id + ":jobs", ...chunk);+ pipeline.sadd("crawl:" + id + ":jobs_qualified", ...chunk);+ }pipeline.expire("crawl:" + id + ":jobs", 24 * 60 * 60);- pipeline.sadd("crawl:" + id + ":jobs_qualified", ...job_ids);pipeline.expire("crawl:" + id + ":jobs_qualified", 24 * 60 * 60);- await pipeline.exec();+ const error = reportPipelineError(await pipeline.exec(), __logger, {+ module: "crawl-redis",+ method: "addCrawlJobs",+ crawlId: id,+ jobCount: job_ids.length,+ });+ if (error) {+ throw new Error("Failed to add crawl jobs: " + error.message, {+ cause: error,+ });+ }}export async function addCrawlJobDone(@@ -204,19 +232,169 @@ export async function addCrawlJobDone(method: "addCrawlJobDone",crawlId: id,});- const pipeline = redisEvictConnection.pipeline();- pipeline.sadd("crawl:" + id + ":jobs_done", job_id);- pipeline.expire("crawl:" + id + ":jobs_done", 24 * 60 * 60);+ // The commands are idempotent, so retry the whole pipeline on failure:+ // callers invoke this on both the success and failure paths of a scrape+ // job, and a throw from the success path would otherwise cascade into the+ // failure path and misreport (and potentially re-run) a finished scrape.+ let lastError: unknown = null;+ for (let attempt = 1; attempt <= 3; attempt++) {+ try {+ const pipeline = redisEvictConnection.pipeline();+ pipeline.sadd("crawl:" + id + ":jobs_done", job_id);+ pipeline.expire("crawl:" + id + ":jobs_done", 24 * 60 * 60);++ if (success) {+ pipeline.zadd(+ "crawl:" + id + ":jobs_donez_ordered",+ Date.now(),+ job_id,+ );+ } else {+ // in case it's already been pushed, make sure it's removed+ pipeline.zrem("crawl:" + id + ":jobs_donez_ordered", job_id);+ }- if (success) {- pipeline.zadd("crawl:" + id + ":jobs_donez_ordered", Date.now(), job_id);- } else {- // in case it's already been pushed, make sure it's removed- pipeline.zrem("crawl:" + id + ":jobs_donez_ordered", job_id);+ pipeline.expire("crawl:" + id + ":jobs_donez_ordered", 24 * 60 * 60);+ const error = firstPipelineError(await pipeline.exec());+ if (error === null) {+ return;+ }+ lastError = error;+ } catch (error) {+ lastError = error;+ }++ __logger.error("Redis pipeline command failed", {+ module: "crawl-redis",+ method: "addCrawlJobDone",+ crawlId: id,+ jobId: job_id,+ attempt,+ error: lastError,+ });++ if (attempt < 3) {+ await new Promise(resolve => setTimeout(resolve, 50 * 2 ** attempt));+ }+ }++ throw new Error("Failed to mark crawl job as done after retries", {+ cause: lastError,+ });+}++const CRAWL_JOB_DONE_REPAIR_KEY = "crawl-job-done-repair";+const CRAWL_JOB_DONE_REPAIR_BATCH = 100;+const CRAWL_JOB_DONE_REPAIR_MAX = 10000;++type CrawlJobDoneRepairEntry = {+ id: string;+ job_id: string;+ success: boolean;+ at: number;+};++/// Best-effort durable record of a crawl completion marker that could not+/// be written even after retries. Without it, an exhausted addCrawlJobDone+/// leaves the crawl's done-set permanently short and the crawl never+/// completes. Drained by repairCrawlJobDoneMarkers, which the NuQ+/// reconciler worker runs every tick. Never throws.+export async function queueCrawlJobDoneRepair(+ id: string,+ job_id: string,+ success: boolean,+ __logger: Logger = _logger,+) {+ const entry: CrawlJobDoneRepairEntry = {+ id,+ job_id,+ success,+ at: Date.now(),+ };+ // The repair store is a hash keyed by crawl+job, so the write is+ // idempotent and retries can never duplicate an entry or displace+ // another crawl's marker.+ for (let attempt = 1; attempt <= 3; attempt++) {+ try {+ await redisEvictConnection.hset(+ CRAWL_JOB_DONE_REPAIR_KEY,+ id + ":" + job_id,+ JSON.stringify(entry),+ );+ return;+ } catch (error) {+ __logger.error("Redis pipeline command failed", {+ module: "crawl-redis",+ method: "queueCrawlJobDoneRepair",+ crawlId: id,+ jobId: job_id,+ attempt,+ error,+ });++ if (attempt < 3) {+ await new Promise(resolve => setTimeout(resolve, 50 * 2 ** attempt));+ }+ }+ }+ // If we get here, Redis is having a sustained outage and nothing local+ // can make the marker recoverable — the canonical logs above are the+ // signal.+}++/// Re-attempts queued crawl completion markers, removing each from the+/// repair hash once it lands. Entries that keep failing stay queued for+/// the next tick. Reads are bounded (HSCAN, not HGETALL) because the hash+/// can accumulate while repairs keep failing, and the scan cursor persists+/// across ticks so a failing early batch cannot starve later entries —+/// every entry is retried once per full scan cycle.+let repairScanCursor = "0";++export async function repairCrawlJobDoneMarkers(__logger: Logger = _logger) {+ const [nextCursor, fields] = await redisEvictConnection.hscan(+ CRAWL_JOB_DONE_REPAIR_KEY,+ repairScanCursor,+ "COUNT",+ CRAWL_JOB_DONE_REPAIR_BATCH,+ );+ // HSCAN returns "0" when the iteration has wrapped; the next tick then+ // starts a fresh cycle and retries the entries that failed this one.+ repairScanCursor = nextCursor;++ for (let i = 0; i + 1 < fields.length; i += 2) {+ const field = fields[i];+ let parsed: CrawlJobDoneRepairEntry;+ try {+ parsed = JSON.parse(fields[i + 1]);+ } catch {+ await redisEvictConnection.hdel(CRAWL_JOB_DONE_REPAIR_KEY, field);+ continue;+ }++ try {+ await addCrawlJobDone(parsed.id, parsed.job_id, parsed.success, __logger);+ await redisEvictConnection.hdel(CRAWL_JOB_DONE_REPAIR_KEY, field);+ } catch {+ // Still failing — leave the entry queued for the next tick.+ }}- pipeline.expire("crawl:" + id + ":jobs_donez_ordered", 24 * 60 * 60);- await pipeline.exec();+ // The backlog bound: a hash over this size means repairs have been+ // failing for a long time and the crawl-completion pipeline needs+ // attention, not silent accumulation.+ const backlog = await redisEvictConnection.hlen(CRAWL_JOB_DONE_REPAIR_KEY);+ if (backlog > CRAWL_JOB_DONE_REPAIR_MAX) {+ __logger.error("Redis pipeline command failed", {+ module: "crawl-redis",+ method: "repairCrawlJobDoneMarkers",+ backlog,+ error: new Error(+ "Crawl completion repair backlog exceeds " ++ CRAWL_JOB_DONE_REPAIR_MAX ++ " entries",+ ),+ });+ }}export async function getDoneJobsOrderedLength(@@ -489,6 +667,7 @@ export async function lockURL(id: string,sc: StoredCrawl,url: string,+ __logger: Logger = _logger,): Promise<boolean> {return await withSpan("firecrawl-redis-lock-url", async span => {const normalizedUrl = normalizeURL(url, sc);@@ -509,26 +688,146 @@ export async function lockURL(}}- const pipeline = redisEvictConnection.pipeline();+ const lockTarget = !sc.crawlerOptions?.deduplicateSimilarURLs+ ? normalizedUrl+ : generateURLPermutations(normalizedUrl)[0].href;++ // The commands are idempotent, so retry the pipeline on failure: the+ // SADD may land before a failing command (e.g. a failed EXPIRE), and+ // giving up then would leave the URL marked visited without being+ // queued — a caller retry would read SADD=0 and skip it. Track whether+ // any attempt reported the SADD as newly added, since a retry after a+ // partial success reads 0.+ let sawNewLock = false;+ let locked = false;+ let lastError: unknown = null;+ for (let attempt = 1; attempt <= 3; attempt++) {+ let attemptError: unknown = null;+ try {+ const pipeline = redisEvictConnection.pipeline();+ pipeline.sadd("crawl:" + id + ":visited", lockTarget);+ pipeline.expire("crawl:" + id + ":visited", 24 * 60 * 60);+ const attemptResults = await pipeline.exec();+ if (attemptResults?.[0]?.[1] === 1) {+ sawNewLock = true;+ }+ attemptError = firstPipelineError(attemptResults);+ } catch (error) {+ attemptError = error;+ }- if (!sc.crawlerOptions?.deduplicateSimilarURLs) {- pipeline.sadd("crawl:" + id + ":visited", normalizedUrl);- } else {- const permutation = generateURLPermutations(normalizedUrl)[0].href;- pipeline.sadd("crawl:" + id + ":visited", permutation);+ if (attemptError === null) {+ locked = true;+ lastError = null;+ break;+ }+ lastError = attemptError;++ __logger.error("Redis pipeline command failed", {+ module: "crawl-redis",+ method: "lockURL",+ crawlId: id,+ url: normalizedUrl,+ attempt,+ error: attemptError,+ });++ if (attempt < 3) {+ await new Promise(resolve => setTimeout(resolve, 50 * 2 ** attempt));+ }}- pipeline.expire("crawl:" + id + ":visited", 24 * 60 * 60);+ if (!locked) {+ throw new Error("URL lock pipeline failed after retries", {+ cause: lastError,+ });+ }- const results = await pipeline.exec();- const saddResult = results?.[0]?.[1] as number;- const res = saddResult !== 0;+ const res = sawNewLock;if (res) {- const uniquePipeline = redisEvictConnection.pipeline();- uniquePipeline.sadd("crawl:" + id + ":visited_unique", normalizedUrl);- uniquePipeline.expire("crawl:" + id + ":visited_unique", 24 * 60 * 60);- await uniquePipeline.exec();+ // The visited SADD has already landed, so a caller retry would skip+ // this URL as already-visited and it would be lost from the crawl.+ // The visited_unique bookkeeping is idempotent — retry it before+ // exposing the failure.+ let lastError: unknown = null;+ for (let attempt = 1; attempt <= 3; attempt++) {+ let attemptError: unknown = null;+ try {+ const uniquePipeline = redisEvictConnection.pipeline();+ uniquePipeline.sadd("crawl:" + id + ":visited_unique", normalizedUrl);+ uniquePipeline.expire(+ "crawl:" + id + ":visited_unique",+ 24 * 60 * 60,+ );+ attemptError = firstPipelineError(await uniquePipeline.exec());+ } catch (error) {+ attemptError = error;+ }++ if (attemptError === null) {+ lastError = null;+ break;+ }+ lastError = attemptError;++ __logger.error("Redis pipeline command failed", {+ module: "crawl-redis",+ method: "lockURL",+ crawlId: id,+ url: normalizedUrl,+ attempt,+ error: attemptError,+ });++ if (attempt < 3) {+ await new Promise(resolve => setTimeout(resolve, 50 * 2 ** attempt));+ }+ }++ if (lastError !== null) {+ // Roll back the locks we just acquired, so a caller retry can+ // re-lock and queue this URL instead of skipping it as+ // already-visited. visited_unique may have landed during a retried+ // attempt whose EXPIRE failed, so remove it too — otherwise the+ // stale member consumes crawl-limit budget for a URL that was+ // never queued. The rollback itself is retried; if it still fails,+ // Redis is having a sustained outage and the canonical logs are+ // the signal.+ for (let attempt = 1; attempt <= 3; attempt++) {+ let rollbackError: unknown = null;+ try {+ const rollback = redisEvictConnection.pipeline();+ rollback.srem("crawl:" + id + ":visited", lockTarget);+ rollback.srem("crawl:" + id + ":visited_unique", normalizedUrl);+ rollbackError = firstPipelineError(await rollback.exec());+ } catch (error) {+ rollbackError = error;+ }++ if (rollbackError === null) {+ break;+ }++ __logger.error("Redis pipeline command failed", {+ module: "crawl-redis",+ method: "lockURL",+ crawlId: id,+ url: normalizedUrl,+ attempt,+ error: rollbackError,+ });++ if (attempt < 3) {+ await new Promise(resolve =>+ setTimeout(resolve, 50 * 2 ** attempt),+ );+ }+ }+ throw new Error("Unique URL lock pipeline failed after retries", {+ cause: lastError,+ });+ }}setSpanAttributes(span, { "crawl.url_locked": res });@@ -556,27 +855,89 @@ export async function lockURLs(// Add to visited_unique setlogger.debug("Locking " + urls.length + " URLs...");- const pipeline = redisEvictConnection.pipeline();- pipeline.sadd("crawl:" + id + ":visited_unique", ...urls);- pipeline.expire("crawl:" + id + ":visited_unique", 24 * 60 * 60);+ const visitedUrls = !sc.crawlerOptions?.deduplicateSimilarURLs+ ? urls+ : urls.map(url => generateURLPermutations(url)[0].href);+ if (sc.crawlerOptions?.deduplicateSimilarURLs) {+ logger.debug("Adding URL permutations...", { count: visitedUrls.length });+ }- if (!sc.crawlerOptions?.deduplicateSimilarURLs) {- pipeline.sadd("crawl:" + id + ":visited", ...urls);- } else {- const allPermutations = urls.map(- url => generateURLPermutations(url)[0].href,- );- logger.debug("Adding " + allPermutations.length + " URL permutations...");- pipeline.sadd("crawl:" + id + ":visited", ...allPermutations);+ // The commands are idempotent, so retry the pipeline on failure: an+ // early chunk may land before a later one fails, and giving up then+ // leaves those URLs marked visited without being queued — a caller retry+ // would read them as already locked and skip them.+ let results: [Error | null, unknown][] | null = null;+ let visitedChunkStartIndexes: number[] = [];+ let lastError: unknown = null;+ for (let attempt = 1; attempt <= 3; attempt++) {+ let attemptError: unknown = null;+ try {+ const pipeline = redisEvictConnection.pipeline();+ for (let i = 0; i < urls.length; i += REDIS_COMMAND_ARG_CHUNK_SIZE) {+ pipeline.sadd(+ "crawl:" + id + ":visited_unique",+ ...urls.slice(i, i + REDIS_COMMAND_ARG_CHUNK_SIZE),+ );+ }+ pipeline.expire("crawl:" + id + ":visited_unique", 24 * 60 * 60);++ visitedChunkStartIndexes = [];+ for (+ let i = 0;+ i < visitedUrls.length;+ i += REDIS_COMMAND_ARG_CHUNK_SIZE+ ) {+ visitedChunkStartIndexes.push(pipeline.length);+ pipeline.sadd(+ "crawl:" + id + ":visited",+ ...visitedUrls.slice(i, i + REDIS_COMMAND_ARG_CHUNK_SIZE),+ );+ }++ pipeline.expire("crawl:" + id + ":visited", 24 * 60 * 60);++ const attemptResults = await pipeline.exec();+ attemptError = firstPipelineError(attemptResults);+ if (attemptError === null) {+ results = attemptResults;+ }+ } catch (error) {+ attemptError = error;+ }++ if (attemptError === null) {+ lastError = null;+ break;+ }+ lastError = attemptError;++ logger.error("Redis pipeline command failed", {+ urlCount: urls.length,+ attempt,+ error: attemptError,+ });++ if (attempt < 3) {+ await new Promise(resolve => setTimeout(resolve, 50 * 2 ** attempt));+ }}- pipeline.expire("crawl:" + id + ":visited", 24 * 60 * 60);+ if (results === null) {+ throw new Error("Failed to lock URLs after retries", {+ cause: lastError,+ });+ }- const results = await pipeline.exec();- const saddResult = results?.[2]?.[1] as number;+ // If an earlier attempt partially landed, the clean attempt's SADD counts+ // under-report (members already present), so res can be false even though+ // every URL is locked. Callers only use this for logging.+ const saddResult = visitedChunkStartIndexes.reduce(+ (total, index) => total + ((results?.[index]?.[1] as number) ?? 0),+ 0,+ );const res = saddResult === urls.length;- logger.debug("lockURLs final result: " + res, { res });+ logger.debug("lockURLs final result", { res });return res;}diff --git a/apps/api/src/lib/redis-pipeline.ts b/apps/api/src/lib/redis-pipeline.tsnew file mode 100644index 000000000..1ebd1c197--- /dev/null+++ b/apps/api/src/lib/redis-pipeline.ts@@ -0,0 +1,36 @@+// ioredis pipelines (and MULTI transactions) do not reject on per-command+// failures: exec() resolves with an array of [err, result] tuples and the+// caller must inspect it. These helpers implement the shared handling for+// that, so a failing command inside a pipeline is never silently dropped.+//+// Every detection site logs the same canonical line —+// "Redis pipeline command failed"+// — so all of these events are filterable by that message across services.++type PipelineResults = [Error | null, unknown][] | null;++// Dragonfly rejects commands with more than 2^16 arguments ("invalid+// multibulk length"), so variadic commands over unbounded collections must+// be chunked well below that limit.+export const REDIS_COMMAND_ARG_CHUNK_SIZE = 16000;++export function firstPipelineError(results: PipelineResults): Error | null {+ const err = (results ?? []).find(([e]) => e)?.[0];+ return err ?? null;+}++// Logs the canonical line when a pipeline result contains a command error,+// and returns that error so the caller can decide whether to throw or+// continue. `context` should identify the site (module, method, and any+// relevant IDs); the error itself is added automatically.+export function reportPipelineError(+ results: PipelineResults,+ logger: { error: (message: string, meta: Record<string, unknown>) => void },+ context: Record<string, unknown>,+): Error | null {+ const error = firstPipelineError(results);+ if (error) {+ logger.error("Redis pipeline command failed", { ...context, error });+ }+ return error;+}diff --git a/apps/api/src/lib/threat-protection/providers/zscaler/lookup-queue.ts b/apps/api/src/lib/threat-protection/providers/zscaler/lookup-queue.tsindex 5ada6dd80..3ae2bab79 100644--- a/apps/api/src/lib/threat-protection/providers/zscaler/lookup-queue.ts+++ b/apps/api/src/lib/threat-protection/providers/zscaler/lookup-queue.ts@@ -4,6 +4,7 @@ import { logger as _logger } from "../../../logger";import { redisRateLimitClient } from "../../../../services/rate-limiter";import { redlock } from "../../../../services/redlock";import { getOrgZscalerCredentials } from "../../store";+import { firstPipelineError } from "../../../redis-pipeline";import {ZscalerError,zscalerLookupUrls,@@ -310,20 +311,47 @@ function isExpired(entry: QueueEntry): boolean {);}+// Never throws: the reply write is an idempotent SET with an+// already-computed payload, so it retries in place — requeueing the entries+// would re-spend the tenant's urlLookup budget on a call that already+// succeeded. If every attempt fails, requesters exit via their wait timeout+// and the org's failurePolicy, which is the designed backstop.async function replyAll(entries: QueueEntry[],payloadFor: (entry: QueueEntry) => ReplyPayload,): Promise<void> {- const pipeline = redisRateLimitClient.pipeline();- for (const entry of entries) {- pipeline.set(- replyKey(entry.id),- JSON.stringify(payloadFor(entry)),- "EX",- REPLY_TTL_SECONDS,- );+ for (let attempt = 1; attempt <= 3; attempt++) {+ let error: unknown = null;+ try {+ const pipeline = redisRateLimitClient.pipeline();+ for (const entry of entries) {+ pipeline.set(+ replyKey(entry.id),+ JSON.stringify(payloadFor(entry)),+ "EX",+ REPLY_TTL_SECONDS,+ );+ }+ error = firstPipelineError(await pipeline.exec());+ } catch (e) {+ error = e;+ }++ if (error === null) {+ return;+ }++ logger.error("Redis pipeline command failed", {+ method: "replyAll",+ entries: entries.length,+ attempt,+ error,+ });++ if (attempt < 3) {+ await sleep(50 * 2 ** attempt);+ }}- await pipeline.exec();}async function drainQueue(orgId: string): Promise<void> {@@ -357,7 +385,7 @@ async function drainQueue(orgId: string): Promise<void> {await replyAll(expired, () => ({status: "error",message: "The lookup request expired before the batch was sent",- })).catch(() => {});+ }));entries = entries.filter(entry => !isExpired(entry));if (entries.length === 0) continue;}@@ -408,31 +436,17 @@ async function drainQueue(orgId: string): Promise<void> {await replyAll(entries, () => ({status: "error",message: "Zscaler lookup drainer failed before the batch was sent",- })).catch(() => {});+ }));throw error;}const uniqueUrls = [...new Set(entries.map(entry => entry.url))];+ let byUrl: Map<string, ZscalerUrlLookupResult>;try {const results = await zscalerLookupUrls(credentials, uniqueUrls, {signal: AbortSignal.timeout(LOOKUP_CALL_TIMEOUT_MS),});- const byUrl = new Map(results.map(result => [result.url, result]));- await replyAll(entries, entry => {- const result = byUrl.get(entry.url);- if (!result) {- return {- status: "error",- message: "Zscaler returned no verdict for this URL",- };- }- return {- status: "ok",- urlClassifications: result.urlClassifications,- urlClassificationsWithSecurityAlert:- result.urlClassificationsWithSecurityAlert,- };- });+ byUrl = new Map(results.map(result => [result.url, result]));} catch (error) {const zscalerError =error instanceof ZscalerError@@ -461,7 +475,26 @@ async function drainQueue(orgId: string): Promise<void> {status: "error",message: zscalerError.message,}));+ continue;}++ // Reply outside the lookup try/catch: a reply-write failure is not a+ // lookup failure and must not be handled as one.+ await replyAll(entries, entry => {+ const result = byUrl.get(entry.url);+ if (!result) {+ return {+ status: "error",+ message: "Zscaler returned no verdict for this URL",+ };+ }+ return {+ status: "ok",+ urlClassifications: result.urlClassifications,+ urlClassificationsWithSecurityAlert:+ result.urlClassificationsWithSecurityAlert,+ };+ });}} finally {await lock.release().catch(() => {});diff --git a/apps/api/src/services/index-cache.ts b/apps/api/src/services/index-cache.tsindex 3786d526e..d1569b608 100644--- a/apps/api/src/services/index-cache.ts+++ b/apps/api/src/services/index-cache.ts@@ -2,6 +2,7 @@ import crypto from "crypto";import IORedis from "ioredis";import { config } from "../config";import { logger as _logger } from "../lib/logger";+import { reportPipelineError } from "../lib/redis-pipeline";import {indexCacheErrorCounter,indexCacheReadDuration,@@ -222,6 +223,23 @@ export async function upsertCachedIndexEntries(pipeline.del(negKeyFor(key));pipeline.hlen(key);const results = await pipeline.exec();+ reportPipelineError(results, logger, {+ module: "index-cache",+ method: "upsertCachedIndexEntries",+ key,+ });++ // If invalidating the negative marker failed, a surviving marker+ // would falsely keep proving "nothing inserted since" — retry the DEL+ // on its own. This is unconditional because deleting the marker is+ // always safe (it only costs a DB read on the next lookup), and a+ // connection failure can report the HSET as failed even when Redis+ // applied it. A throw here is caught below and logged as a write+ // failure.+ if (results?.[2]?.[0]) {+ await client.del(negKeyFor(key));+ }+const hlen = results?.[3]?.[1];if (typeof hlen === "number" && hlen > ENTRY_CAP) {const raw = await client.hgetall(key);diff --git a/apps/api/src/services/worker/nuq-reconciler-worker.ts b/apps/api/src/services/worker/nuq-reconciler-worker.tsindex 796ecb5f8..865da6549 100644--- a/apps/api/src/services/worker/nuq-reconciler-worker.ts+++ b/apps/api/src/services/worker/nuq-reconciler-worker.ts@@ -4,6 +4,7 @@ import "../sentry";import { setSentryServiceTag } from "../sentry";import { logger as _logger } from "../../lib/logger";import { reconcileConcurrencyQueue } from "../../lib/concurrency-queue-reconciler";+import { repairCrawlJobDoneMarkers } from "../../lib/crawl-redis";import { Counter, register } from "prom-client";import Express from "express";@@ -86,6 +87,12 @@ const reconcilerJobsRecoveredTotal = new Counter({if (!reconcilerInFlight) {reconcilerInFlight = true;+ try {+ await repairCrawlJobDoneMarkers(_logger);+ } catch (error) {+ _logger.error("Crawl completion marker repair run failed", { error });+ }+try {const summary = await reconcileConcurrencyQueue({logger: _logger,diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 323f6575b..87b838f69 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -18,6 +18,7 @@ import {addCrawlJobs,addCrawlJobDone,crawlToCrawler,+ queueCrawlJobDoneRepair,recordRobotsBlocked,recordThreatBlocked,finishCrawlKickoff,@@ -834,7 +835,22 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {await recordMonitorScrapeSuccess(job, doc);logger.debug("Declaring job as done...");- await addCrawlJobDone(job.data.crawl_id, job.id, true, logger);+ try {+ await addCrawlJobDone(job.data.crawl_id, job.id, true, logger);+ } catch (e) {+ // The scrape succeeded and its success webhook already went out —+ // a bookkeeping failure must not route this job through the+ // failure path (contradictory failure webhook, misrecorded job).+ // Already logged canonically inside addCrawlJobDone.+ logger.error("Failed to mark successful crawl job as done", {+ crawlId: job.data.crawl_id,+ jobId: job.id,+ error: e,+ });+ // Durable fallback so the crawl's completion marker is retried by+ // the reconciler instead of being lost for good.+ await queueCrawlJobDoneRepair(job.data.crawl_id, job.id, true, logger);+ }} else {try {signal?.throwIfAborted();@@ -944,7 +960,18 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;logger.debug("Declaring job as done...");- await addCrawlJobDone(job.data.crawl_id, job.id, false, logger);+ try {+ await addCrawlJobDone(job.data.crawl_id, job.id, false, logger);+ } catch (e) {+ // Already logged canonically inside addCrawlJobDone; a throw here+ // must not escape the error handler and fail the job a second time.+ logger.error("Failed to declare failed crawl job as done", {+ crawlId: job.data.crawl_id,+ jobId: job.id,+ error: e,+ });+ await queueCrawlJobDoneRepair(job.data.crawl_id, job.id, false, logger);+ }await redisEvictConnection.srem("crawl:" + job.data.crawl_id + ":visited_unique",normalizeURL(job.data.url, sc),