Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): check Redis pipeline results instead of swallowing command errors (#4557)

Gergő Móricze4ac89b02568
diff --git a/apps/api/src/__tests__/snips/index-cache.test.ts b/apps/api/src/__tests__/snips/index-cache.test.ts
index 9d4366e41..ecf11dfdb 100644
--- a/apps/api/src/__tests__/snips/index-cache.test.ts
+++ b/apps/api/src/__tests__/snips/index-cache.test.ts
@@ -273,3 +273,68 @@ describe("index cache fail-open", () => {
).resolves.toBeUndefined();
}, 5000);
});
+
+describe("upsertCachedIndexEntries pipeline error handling", () => {
+ function fakeClientWithResults(results: [Error | null, unknown][]) {
+ const delCalls: string[] = [];
+ const client: any = {
+ pipeline: () => {
+ const pipeline: any = {
+ hset: () => pipeline,
+ expire: () => pipeline,
+ del: () => pipeline,
+ hlen: () => pipeline,
+ exec: async () => results,
+ };
+ return pipeline;
+ },
+ del: async (key: string) => {
+ delCalls.push(key);
+ return 1;
+ },
+ };
+ return { client, delCalls };
+ }
+
+ it("retries the negative-marker DEL when the HSET landed but the DEL failed", async () => {
+ const { client, delCalls } = fakeClientWithResults([
+ [null, 1], // hset
+ [null, 1], // expire
+ [new Error("boom"), undefined], // del negative marker
+ [null, 1], // hlen
+ ]);
+
+ await upsertCachedIndexEntries("idxc:test", [entry()], undefined, client);
+
+ expect(delCalls).toEqual(["idxcneg:test"]);
+ });
+
+ it("still retries the negative-marker DEL when the HSET failed", async () => {
+ // A connection failure can report the HSET as failed even when Redis
+ // applied it, and deleting the marker is always safe (it only costs a
+ // DB read on the next lookup).
+ const { client, delCalls } = fakeClientWithResults([
+ [new Error("boom"), undefined], // hset
+ [null, 1], // expire
+ [new Error("boom"), undefined], // del negative marker
+ [null, 1], // hlen
+ ]);
+
+ await upsertCachedIndexEntries("idxc:test", [entry()], undefined, client);
+
+ expect(delCalls).toEqual(["idxcneg:test"]);
+ });
+
+ it("does not retry the negative-marker DEL when it succeeded", async () => {
+ const { client, delCalls } = fakeClientWithResults([
+ [null, 1], // hset
+ [null, 1], // expire
+ [null, 1], // del negative marker
+ [null, 1], // hlen
+ ]);
+
+ await upsertCachedIndexEntries("idxc:test", [entry()], undefined, client);
+
+ expect(delCalls).toEqual([]);
+ });
+});
diff --git a/apps/api/src/controllers/v2/parse-upload.ts b/apps/api/src/controllers/v2/parse-upload.ts
index e4f1acde6..d3f204ada 100644
--- a/apps/api/src/controllers/v2/parse-upload.ts
+++ b/apps/api/src/controllers/v2/parse-upload.ts
@@ -11,6 +11,7 @@ import { getRedisConnection } from "../../services/queue-service";
import { RequestWithAuth, UploadedParseFile } from "./types";
import { detectUploadedFileKind, getSupportedParseFileTypes } from "./parse";
import { isImageOcrEnabled } from "../../lib/image-ocr-gate";
+import { reportPipelineError } from "../../lib/redis-pipeline";
const PARSE_UPLOAD_MAX_BYTES = 50 * 1024 * 1024;
const PARSE_UPLOAD_TTL_MS = 10 * 60 * 1000;
@@ -226,12 +227,25 @@ async function reserveUnparsedUploadRef(teamId: string, uploadId: string) {
async function releaseUnparsedUploadRef(teamId: string, uploadId: string) {
const cutoff = Date.now() - PARSE_UPLOAD_UNPARSED_WINDOW_MS;
const key = getUnparsedUploadsKey(teamId);
- await getRedisConnection()
+ const results = await getRedisConnection()
.pipeline()
.zremrangebyscore(key, "-inf", cutoff)
.zrem(key, uploadId)
.expire(key, PARSE_UPLOAD_UNPARSED_TTL_SECONDS)
.exec();
+ // Both call sites catch and log a throw here; a leaked ref self-expires
+ // via TTL.
+ const error = reportPipelineError(results, _logger, {
+ module: "parse-upload",
+ method: "releaseUnparsedUploadRef",
+ teamId,
+ uploadId,
+ });
+ if (error) {
+ throw new Error("Failed to release unparsed upload ref: " + error.message, {
+ cause: error,
+ });
+ }
}
async function reserveUnparsedUploadRefOrRespond(
diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.ts
index 246e376f0..db98cd744 100644
--- a/apps/api/src/lib/concurrency-limit.ts
+++ b/apps/api/src/lib/concurrency-limit.ts
@@ -19,6 +19,7 @@ import {
removeConcurrencyLimitActiveJob,
} from "./concurrency-redis";
import { autumnService } from "../services/autumn/autumn.service";
+import { reportPipelineError } from "./redis-pipeline";
// Fallback when Autumn can't give us a concurrency value.
const DEFAULT_CONCURRENCY_LIMIT = 2;
@@ -83,7 +84,15 @@ export async function removeConcurrencyLimitedJobs(
for (const id of chunk) {
pipeline.del(constructJobKey(id));
}
- await pipeline.exec();
+ // Do not throw on command errors: cancel has already been recorded on
+ // the crawl, and the stale entries self-expire via their PX timeout.
+ // But never let the failure pass silently.
+ reportPipelineError(await pipeline.exec(), logger, {
+ module: "concurrency-limit",
+ method: "removeConcurrencyLimitedJobs",
+ teamId: team_id,
+ jobCount: chunk.length,
+ });
}
}
@@ -143,7 +152,16 @@ export async function pushConcurrencyLimitedJobs(
pipeline.zadd(queueKey, ...zaddArgs);
pipeline.sadd("concurrency-limit-queues", queueKey);
- await pipeline.exec();
+ // Do not throw on command errors: the jobs are already durable in the
+ // NuQ backlog, and the concurrency-queue reconciler requeues anything
+ // missing from this derived Redis index on its next run. But never let
+ // the failure pass silently.
+ reportPipelineError(await pipeline.exec(), logger, {
+ module: "concurrency-limit",
+ method: "pushConcurrencyLimitedJobs",
+ teamId: team_id,
+ jobCount: jobs.length,
+ });
}
export async function getConcurrencyLimitedJobs(team_id: string) {
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
index eb9d8cb7a..f9a2b1ee1 100644
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@ -8,6 +8,11 @@ import { getAdjustedMaxDepth } from "../scraper/WebScraper/utils/maxDepthUtils";
import type { Logger } from "winston";
import { withSpan, setSpanAttributes } from "./otel-tracer";
import { getScrapeZDR, getIgnoreRobots } from "./zdr-helpers";
+import {
+ firstPipelineError,
+ reportPipelineError,
+ REDIS_COMMAND_ARG_CHUNK_SIZE,
+} from "./redis-pipeline";
export type StoredCrawl = {
originUrl?: string;
@@ -167,7 +172,17 @@ export async function addCrawlJob(
pipeline.expire("crawl:" + id + ":jobs", 24 * 60 * 60);
pipeline.sadd("crawl:" + id + ":jobs_qualified", job_id);
pipeline.expire("crawl:" + id + ":jobs_qualified", 24 * 60 * 60);
- await pipeline.exec();
+ const error = reportPipelineError(await pipeline.exec(), __logger, {
+ module: "crawl-redis",
+ method: "addCrawlJob",
+ crawlId: id,
+ jobId: job_id,
+ });
+ if (error) {
+ throw new Error("Failed to add crawl job: " + error.message, {
+ cause: error,
+ });
+ }
});
}
@@ -185,11 +200,24 @@ export async function addCrawlJobs(
crawlId: id,
});
const pipeline = redisEvictConnection.pipeline();
- pipeline.sadd("crawl:" + id + ":jobs", ...job_ids);
+ for (let i = 0; i < job_ids.length; i += REDIS_COMMAND_ARG_CHUNK_SIZE) {
+ const chunk = job_ids.slice(i, i + REDIS_COMMAND_ARG_CHUNK_SIZE);
+ pipeline.sadd("crawl:" + id + ":jobs", ...chunk);
+ pipeline.sadd("crawl:" + id + ":jobs_qualified", ...chunk);
+ }
pipeline.expire("crawl:" + id + ":jobs", 24 * 60 * 60);
- pipeline.sadd("crawl:" + id + ":jobs_qualified", ...job_ids);
pipeline.expire("crawl:" + id + ":jobs_qualified", 24 * 60 * 60);
- await pipeline.exec();
+ const error = reportPipelineError(await pipeline.exec(), __logger, {
+ module: "crawl-redis",
+ method: "addCrawlJobs",
+ crawlId: id,
+ jobCount: job_ids.length,
+ });
+ if (error) {
+ throw new Error("Failed to add crawl jobs: " + error.message, {
+ cause: error,
+ });
+ }
}
export async function addCrawlJobDone(
@@ -204,19 +232,169 @@ export async function addCrawlJobDone(
method: "addCrawlJobDone",
crawlId: id,
});
- const pipeline = redisEvictConnection.pipeline();
- pipeline.sadd("crawl:" + id + ":jobs_done", job_id);
- pipeline.expire("crawl:" + id + ":jobs_done", 24 * 60 * 60);
+ // The commands are idempotent, so retry the whole pipeline on failure:
+ // callers invoke this on both the success and failure paths of a scrape
+ // job, and a throw from the success path would otherwise cascade into the
+ // failure path and misreport (and potentially re-run) a finished scrape.
+ let lastError: unknown = null;
+ for (let attempt = 1; attempt <= 3; attempt++) {
+ try {
+ const pipeline = redisEvictConnection.pipeline();
+ pipeline.sadd("crawl:" + id + ":jobs_done", job_id);
+ pipeline.expire("crawl:" + id + ":jobs_done", 24 * 60 * 60);
+
+ if (success) {
+ pipeline.zadd(
+ "crawl:" + id + ":jobs_donez_ordered",
+ Date.now(),
+ job_id,
+ );
+ } else {
+ // in case it's already been pushed, make sure it's removed
+ pipeline.zrem("crawl:" + id + ":jobs_donez_ordered", job_id);
+ }
- if (success) {
- pipeline.zadd("crawl:" + id + ":jobs_donez_ordered", Date.now(), job_id);
- } else {
- // in case it's already been pushed, make sure it's removed
- pipeline.zrem("crawl:" + id + ":jobs_donez_ordered", job_id);
+ pipeline.expire("crawl:" + id + ":jobs_donez_ordered", 24 * 60 * 60);
+ const error = firstPipelineError(await pipeline.exec());
+ if (error === null) {
+ return;
+ }
+ lastError = error;
+ } catch (error) {
+ lastError = error;
+ }
+
+ __logger.error("Redis pipeline command failed", {
+ module: "crawl-redis",
+ method: "addCrawlJobDone",
+ crawlId: id,
+ jobId: job_id,
+ attempt,
+ error: lastError,
+ });
+
+ if (attempt < 3) {
+ await new Promise(resolve => setTimeout(resolve, 50 * 2 ** attempt));
+ }
+ }
+
+ throw new Error("Failed to mark crawl job as done after retries", {
+ cause: lastError,
+ });
+}
+
+const CRAWL_JOB_DONE_REPAIR_KEY = "crawl-job-done-repair";
+const CRAWL_JOB_DONE_REPAIR_BATCH = 100;
+const CRAWL_JOB_DONE_REPAIR_MAX = 10000;
+
+type CrawlJobDoneRepairEntry = {
+ id: string;
+ job_id: string;
+ success: boolean;
+ at: number;
+};
+
+/// Best-effort durable record of a crawl completion marker that could not
+/// be written even after retries. Without it, an exhausted addCrawlJobDone
+/// leaves the crawl's done-set permanently short and the crawl never
+/// completes. Drained by repairCrawlJobDoneMarkers, which the NuQ
+/// reconciler worker runs every tick. Never throws.
+export async function queueCrawlJobDoneRepair(
+ id: string,
+ job_id: string,
+ success: boolean,
+ __logger: Logger = _logger,
+) {
+ const entry: CrawlJobDoneRepairEntry = {
+ id,
+ job_id,
+ success,
+ at: Date.now(),
+ };
+ // The repair store is a hash keyed by crawl+job, so the write is
+ // idempotent and retries can never duplicate an entry or displace
+ // another crawl's marker.
+ for (let attempt = 1; attempt <= 3; attempt++) {
+ try {
+ await redisEvictConnection.hset(
+ CRAWL_JOB_DONE_REPAIR_KEY,
+ id + ":" + job_id,
+ JSON.stringify(entry),
+ );
+ return;
+ } catch (error) {
+ __logger.error("Redis pipeline command failed", {
+ module: "crawl-redis",
+ method: "queueCrawlJobDoneRepair",
+ crawlId: id,
+ jobId: job_id,
+ attempt,
+ error,
+ });
+
+ if (attempt < 3) {
+ await new Promise(resolve => setTimeout(resolve, 50 * 2 ** attempt));
+ }
+ }
+ }
+ // If we get here, Redis is having a sustained outage and nothing local
+ // can make the marker recoverable — the canonical logs above are the
+ // signal.
+}
+
+/// Re-attempts queued crawl completion markers, removing each from the
+/// repair hash once it lands. Entries that keep failing stay queued for
+/// the next tick. Reads are bounded (HSCAN, not HGETALL) because the hash
+/// can accumulate while repairs keep failing, and the scan cursor persists
+/// across ticks so a failing early batch cannot starve later entries —
+/// every entry is retried once per full scan cycle.
+let repairScanCursor = "0";
+
+export async function repairCrawlJobDoneMarkers(__logger: Logger = _logger) {
+ const [nextCursor, fields] = await redisEvictConnection.hscan(
+ CRAWL_JOB_DONE_REPAIR_KEY,
+ repairScanCursor,
+ "COUNT",
+ CRAWL_JOB_DONE_REPAIR_BATCH,
+ );
+ // HSCAN returns "0" when the iteration has wrapped; the next tick then
+ // starts a fresh cycle and retries the entries that failed this one.
+ repairScanCursor = nextCursor;
+
+ for (let i = 0; i + 1 < fields.length; i += 2) {
+ const field = fields[i];
+ let parsed: CrawlJobDoneRepairEntry;
+ try {
+ parsed = JSON.parse(fields[i + 1]);
+ } catch {
+ await redisEvictConnection.hdel(CRAWL_JOB_DONE_REPAIR_KEY, field);
+ continue;
+ }
+
+ try {
+ await addCrawlJobDone(parsed.id, parsed.job_id, parsed.success, __logger);
+ await redisEvictConnection.hdel(CRAWL_JOB_DONE_REPAIR_KEY, field);
+ } catch {
+ // Still failing — leave the entry queued for the next tick.
+ }
}
- pipeline.expire("crawl:" + id + ":jobs_donez_ordered", 24 * 60 * 60);
- await pipeline.exec();
+ // The backlog bound: a hash over this size means repairs have been
+ // failing for a long time and the crawl-completion pipeline needs
+ // attention, not silent accumulation.
+ const backlog = await redisEvictConnection.hlen(CRAWL_JOB_DONE_REPAIR_KEY);
+ if (backlog > CRAWL_JOB_DONE_REPAIR_MAX) {
+ __logger.error("Redis pipeline command failed", {
+ module: "crawl-redis",
+ method: "repairCrawlJobDoneMarkers",
+ backlog,
+ error: new Error(
+ "Crawl completion repair backlog exceeds " +
+ CRAWL_JOB_DONE_REPAIR_MAX +
+ " entries",
+ ),
+ });
+ }
}
export async function getDoneJobsOrderedLength(
@@ -489,6 +667,7 @@ export async function lockURL(
id: string,
sc: StoredCrawl,
url: string,
+ __logger: Logger = _logger,
): Promise<boolean> {
return await withSpan("firecrawl-redis-lock-url", async span => {
const normalizedUrl = normalizeURL(url, sc);
@@ -509,26 +688,146 @@ export async function lockURL(
}
}
- const pipeline = redisEvictConnection.pipeline();
+ const lockTarget = !sc.crawlerOptions?.deduplicateSimilarURLs
+ ? normalizedUrl
+ : generateURLPermutations(normalizedUrl)[0].href;
+
+ // The commands are idempotent, so retry the pipeline on failure: the
+ // SADD may land before a failing command (e.g. a failed EXPIRE), and
+ // giving up then would leave the URL marked visited without being
+ // queued — a caller retry would read SADD=0 and skip it. Track whether
+ // any attempt reported the SADD as newly added, since a retry after a
+ // partial success reads 0.
+ let sawNewLock = false;
+ let locked = false;
+ let lastError: unknown = null;
+ for (let attempt = 1; attempt <= 3; attempt++) {
+ let attemptError: unknown = null;
+ try {
+ const pipeline = redisEvictConnection.pipeline();
+ pipeline.sadd("crawl:" + id + ":visited", lockTarget);
+ pipeline.expire("crawl:" + id + ":visited", 24 * 60 * 60);
+ const attemptResults = await pipeline.exec();
+ if (attemptResults?.[0]?.[1] === 1) {
+ sawNewLock = true;
+ }
+ attemptError = firstPipelineError(attemptResults);
+ } catch (error) {
+ attemptError = error;
+ }
- if (!sc.crawlerOptions?.deduplicateSimilarURLs) {
- pipeline.sadd("crawl:" + id + ":visited", normalizedUrl);
- } else {
- const permutation = generateURLPermutations(normalizedUrl)[0].href;
- pipeline.sadd("crawl:" + id + ":visited", permutation);
+ if (attemptError === null) {
+ locked = true;
+ lastError = null;
+ break;
+ }
+ lastError = attemptError;
+
+ __logger.error("Redis pipeline command failed", {
+ module: "crawl-redis",
+ method: "lockURL",
+ crawlId: id,
+ url: normalizedUrl,
+ attempt,
+ error: attemptError,
+ });
+
+ if (attempt < 3) {
+ await new Promise(resolve => setTimeout(resolve, 50 * 2 ** attempt));
+ }
}
- pipeline.expire("crawl:" + id + ":visited", 24 * 60 * 60);
+ if (!locked) {
+ throw new Error("URL lock pipeline failed after retries", {
+ cause: lastError,
+ });
+ }
- const results = await pipeline.exec();
- const saddResult = results?.[0]?.[1] as number;
- const res = saddResult !== 0;
+ const res = sawNewLock;
if (res) {
- const uniquePipeline = redisEvictConnection.pipeline();
- uniquePipeline.sadd("crawl:" + id + ":visited_unique", normalizedUrl);
- uniquePipeline.expire("crawl:" + id + ":visited_unique", 24 * 60 * 60);
- await uniquePipeline.exec();
+ // The visited SADD has already landed, so a caller retry would skip
+ // this URL as already-visited and it would be lost from the crawl.
+ // The visited_unique bookkeeping is idempotent — retry it before
+ // exposing the failure.
+ let lastError: unknown = null;
+ for (let attempt = 1; attempt <= 3; attempt++) {
+ let attemptError: unknown = null;
+ try {
+ const uniquePipeline = redisEvictConnection.pipeline();
+ uniquePipeline.sadd("crawl:" + id + ":visited_unique", normalizedUrl);
+ uniquePipeline.expire(
+ "crawl:" + id + ":visited_unique",
+ 24 * 60 * 60,
+ );
+ attemptError = firstPipelineError(await uniquePipeline.exec());
+ } catch (error) {
+ attemptError = error;
+ }
+
+ if (attemptError === null) {
+ lastError = null;
+ break;
+ }
+ lastError = attemptError;
+
+ __logger.error("Redis pipeline command failed", {
+ module: "crawl-redis",
+ method: "lockURL",
+ crawlId: id,
+ url: normalizedUrl,
+ attempt,
+ error: attemptError,
+ });
+
+ if (attempt < 3) {
+ await new Promise(resolve => setTimeout(resolve, 50 * 2 ** attempt));
+ }
+ }
+
+ if (lastError !== null) {
+ // Roll back the locks we just acquired, so a caller retry can
+ // re-lock and queue this URL instead of skipping it as
+ // already-visited. visited_unique may have landed during a retried
+ // attempt whose EXPIRE failed, so remove it too — otherwise the
+ // stale member consumes crawl-limit budget for a URL that was
+ // never queued. The rollback itself is retried; if it still fails,
+ // Redis is having a sustained outage and the canonical logs are
+ // the signal.
+ for (let attempt = 1; attempt <= 3; attempt++) {
+ let rollbackError: unknown = null;
+ try {
+ const rollback = redisEvictConnection.pipeline();
+ rollback.srem("crawl:" + id + ":visited", lockTarget);
+ rollback.srem("crawl:" + id + ":visited_unique", normalizedUrl);
+ rollbackError = firstPipelineError(await rollback.exec());
+ } catch (error) {
+ rollbackError = error;
+ }
+
+ if (rollbackError === null) {
+ break;
+ }
+
+ __logger.error("Redis pipeline command failed", {
+ module: "crawl-redis",
+ method: "lockURL",
+ crawlId: id,
+ url: normalizedUrl,
+ attempt,
+ error: rollbackError,
+ });
+
+ if (attempt < 3) {
+ await new Promise(resolve =>
+ setTimeout(resolve, 50 * 2 ** attempt),
+ );
+ }
+ }
+ throw new Error("Unique URL lock pipeline failed after retries", {
+ cause: lastError,
+ });
+ }
}
setSpanAttributes(span, { "crawl.url_locked": res });
@@ -556,27 +855,89 @@ export async function lockURLs(
// Add to visited_unique set
logger.debug("Locking " + urls.length + " URLs...");
- const pipeline = redisEvictConnection.pipeline();
- pipeline.sadd("crawl:" + id + ":visited_unique", ...urls);
- pipeline.expire("crawl:" + id + ":visited_unique", 24 * 60 * 60);
+ const visitedUrls = !sc.crawlerOptions?.deduplicateSimilarURLs
+ ? urls
+ : urls.map(url => generateURLPermutations(url)[0].href);
+ if (sc.crawlerOptions?.deduplicateSimilarURLs) {
+ logger.debug("Adding URL permutations...", { count: visitedUrls.length });
+ }
- if (!sc.crawlerOptions?.deduplicateSimilarURLs) {
- pipeline.sadd("crawl:" + id + ":visited", ...urls);
- } else {
- const allPermutations = urls.map(
- url => generateURLPermutations(url)[0].href,
- );
- logger.debug("Adding " + allPermutations.length + " URL permutations...");
- pipeline.sadd("crawl:" + id + ":visited", ...allPermutations);
+ // The commands are idempotent, so retry the pipeline on failure: an
+ // early chunk may land before a later one fails, and giving up then
+ // leaves those URLs marked visited without being queued — a caller retry
+ // would read them as already locked and skip them.
+ let results: [Error | null, unknown][] | null = null;
+ let visitedChunkStartIndexes: number[] = [];
+ let lastError: unknown = null;
+ for (let attempt = 1; attempt <= 3; attempt++) {
+ let attemptError: unknown = null;
+ try {
+ const pipeline = redisEvictConnection.pipeline();
+ for (let i = 0; i < urls.length; i += REDIS_COMMAND_ARG_CHUNK_SIZE) {
+ pipeline.sadd(
+ "crawl:" + id + ":visited_unique",
+ ...urls.slice(i, i + REDIS_COMMAND_ARG_CHUNK_SIZE),
+ );
+ }
+ pipeline.expire("crawl:" + id + ":visited_unique", 24 * 60 * 60);
+
+ visitedChunkStartIndexes = [];
+ for (
+ let i = 0;
+ i < visitedUrls.length;
+ i += REDIS_COMMAND_ARG_CHUNK_SIZE
+ ) {
+ visitedChunkStartIndexes.push(pipeline.length);
+ pipeline.sadd(
+ "crawl:" + id + ":visited",
+ ...visitedUrls.slice(i, i + REDIS_COMMAND_ARG_CHUNK_SIZE),
+ );
+ }
+
+ pipeline.expire("crawl:" + id + ":visited", 24 * 60 * 60);
+
+ const attemptResults = await pipeline.exec();
+ attemptError = firstPipelineError(attemptResults);
+ if (attemptError === null) {
+ results = attemptResults;
+ }
+ } catch (error) {
+ attemptError = error;
+ }
+
+ if (attemptError === null) {
+ lastError = null;
+ break;
+ }
+ lastError = attemptError;
+
+ logger.error("Redis pipeline command failed", {
+ urlCount: urls.length,
+ attempt,
+ error: attemptError,
+ });
+
+ if (attempt < 3) {
+ await new Promise(resolve => setTimeout(resolve, 50 * 2 ** attempt));
+ }
}
- pipeline.expire("crawl:" + id + ":visited", 24 * 60 * 60);
+ if (results === null) {
+ throw new Error("Failed to lock URLs after retries", {
+ cause: lastError,
+ });
+ }
- const results = await pipeline.exec();
- const saddResult = results?.[2]?.[1] as number;
+ // If an earlier attempt partially landed, the clean attempt's SADD counts
+ // under-report (members already present), so res can be false even though
+ // every URL is locked. Callers only use this for logging.
+ const saddResult = visitedChunkStartIndexes.reduce(
+ (total, index) => total + ((results?.[index]?.[1] as number) ?? 0),
+ 0,
+ );
const res = saddResult === urls.length;
- logger.debug("lockURLs final result: " + res, { res });
+ logger.debug("lockURLs final result", { res });
return res;
}
diff --git a/apps/api/src/lib/redis-pipeline.ts b/apps/api/src/lib/redis-pipeline.ts
new file mode 100644
index 000000000..1ebd1c197
--- /dev/null
+++ b/apps/api/src/lib/redis-pipeline.ts
@@ -0,0 +1,36 @@
+// ioredis pipelines (and MULTI transactions) do not reject on per-command
+// failures: exec() resolves with an array of [err, result] tuples and the
+// caller must inspect it. These helpers implement the shared handling for
+// that, so a failing command inside a pipeline is never silently dropped.
+//
+// Every detection site logs the same canonical line —
+// "Redis pipeline command failed"
+// — so all of these events are filterable by that message across services.
+
+type PipelineResults = [Error | null, unknown][] | null;
+
+// Dragonfly rejects commands with more than 2^16 arguments ("invalid
+// multibulk length"), so variadic commands over unbounded collections must
+// be chunked well below that limit.
+export const REDIS_COMMAND_ARG_CHUNK_SIZE = 16000;
+
+export function firstPipelineError(results: PipelineResults): Error | null {
+ const err = (results ?? []).find(([e]) => e)?.[0];
+ return err ?? null;
+}
+
+// Logs the canonical line when a pipeline result contains a command error,
+// and returns that error so the caller can decide whether to throw or
+// continue. `context` should identify the site (module, method, and any
+// relevant IDs); the error itself is added automatically.
+export function reportPipelineError(
+ results: PipelineResults,
+ logger: { error: (message: string, meta: Record<string, unknown>) => void },
+ context: Record<string, unknown>,
+): Error | null {
+ const error = firstPipelineError(results);
+ if (error) {
+ logger.error("Redis pipeline command failed", { ...context, error });
+ }
+ return error;
+}
diff --git a/apps/api/src/lib/threat-protection/providers/zscaler/lookup-queue.ts b/apps/api/src/lib/threat-protection/providers/zscaler/lookup-queue.ts
index 5ada6dd80..3ae2bab79 100644
--- a/apps/api/src/lib/threat-protection/providers/zscaler/lookup-queue.ts
+++ b/apps/api/src/lib/threat-protection/providers/zscaler/lookup-queue.ts
@@ -4,6 +4,7 @@ import { logger as _logger } from "../../../logger";
import { redisRateLimitClient } from "../../../../services/rate-limiter";
import { redlock } from "../../../../services/redlock";
import { getOrgZscalerCredentials } from "../../store";
+import { firstPipelineError } from "../../../redis-pipeline";
import {
ZscalerError,
zscalerLookupUrls,
@@ -310,20 +311,47 @@ function isExpired(entry: QueueEntry): boolean {
);
}
+// Never throws: the reply write is an idempotent SET with an
+// already-computed payload, so it retries in place — requeueing the entries
+// would re-spend the tenant's urlLookup budget on a call that already
+// succeeded. If every attempt fails, requesters exit via their wait timeout
+// and the org's failurePolicy, which is the designed backstop.
async function replyAll(
entries: QueueEntry[],
payloadFor: (entry: QueueEntry) => ReplyPayload,
): Promise<void> {
- const pipeline = redisRateLimitClient.pipeline();
- for (const entry of entries) {
- pipeline.set(
- replyKey(entry.id),
- JSON.stringify(payloadFor(entry)),
- "EX",
- REPLY_TTL_SECONDS,
- );
+ for (let attempt = 1; attempt <= 3; attempt++) {
+ let error: unknown = null;
+ try {
+ const pipeline = redisRateLimitClient.pipeline();
+ for (const entry of entries) {
+ pipeline.set(
+ replyKey(entry.id),
+ JSON.stringify(payloadFor(entry)),
+ "EX",
+ REPLY_TTL_SECONDS,
+ );
+ }
+ error = firstPipelineError(await pipeline.exec());
+ } catch (e) {
+ error = e;
+ }
+
+ if (error === null) {
+ return;
+ }
+
+ logger.error("Redis pipeline command failed", {
+ method: "replyAll",
+ entries: entries.length,
+ attempt,
+ error,
+ });
+
+ if (attempt < 3) {
+ await sleep(50 * 2 ** attempt);
+ }
}
- await pipeline.exec();
}
async function drainQueue(orgId: string): Promise<void> {
@@ -357,7 +385,7 @@ async function drainQueue(orgId: string): Promise<void> {
await replyAll(expired, () => ({
status: "error",
message: "The lookup request expired before the batch was sent",
- })).catch(() => {});
+ }));
entries = entries.filter(entry => !isExpired(entry));
if (entries.length === 0) continue;
}
@@ -408,31 +436,17 @@ async function drainQueue(orgId: string): Promise<void> {
await replyAll(entries, () => ({
status: "error",
message: "Zscaler lookup drainer failed before the batch was sent",
- })).catch(() => {});
+ }));
throw error;
}
const uniqueUrls = [...new Set(entries.map(entry => entry.url))];
+ let byUrl: Map<string, ZscalerUrlLookupResult>;
try {
const results = await zscalerLookupUrls(credentials, uniqueUrls, {
signal: AbortSignal.timeout(LOOKUP_CALL_TIMEOUT_MS),
});
- const byUrl = new Map(results.map(result => [result.url, result]));
- await replyAll(entries, entry => {
- const result = byUrl.get(entry.url);
- if (!result) {
- return {
- status: "error",
- message: "Zscaler returned no verdict for this URL",
- };
- }
- return {
- status: "ok",
- urlClassifications: result.urlClassifications,
- urlClassificationsWithSecurityAlert:
- result.urlClassificationsWithSecurityAlert,
- };
- });
+ byUrl = new Map(results.map(result => [result.url, result]));
} catch (error) {
const zscalerError =
error instanceof ZscalerError
@@ -461,7 +475,26 @@ async function drainQueue(orgId: string): Promise<void> {
status: "error",
message: zscalerError.message,
}));
+ continue;
}
+
+ // Reply outside the lookup try/catch: a reply-write failure is not a
+ // lookup failure and must not be handled as one.
+ await replyAll(entries, entry => {
+ const result = byUrl.get(entry.url);
+ if (!result) {
+ return {
+ status: "error",
+ message: "Zscaler returned no verdict for this URL",
+ };
+ }
+ return {
+ status: "ok",
+ urlClassifications: result.urlClassifications,
+ urlClassificationsWithSecurityAlert:
+ result.urlClassificationsWithSecurityAlert,
+ };
+ });
}
} finally {
await lock.release().catch(() => {});
diff --git a/apps/api/src/services/index-cache.ts b/apps/api/src/services/index-cache.ts
index 3786d526e..d1569b608 100644
--- a/apps/api/src/services/index-cache.ts
+++ b/apps/api/src/services/index-cache.ts
@@ -2,6 +2,7 @@ import crypto from "crypto";
import IORedis from "ioredis";
import { config } from "../config";
import { logger as _logger } from "../lib/logger";
+import { reportPipelineError } from "../lib/redis-pipeline";
import {
indexCacheErrorCounter,
indexCacheReadDuration,
@@ -222,6 +223,23 @@ export async function upsertCachedIndexEntries(
pipeline.del(negKeyFor(key));
pipeline.hlen(key);
const results = await pipeline.exec();
+ reportPipelineError(results, logger, {
+ module: "index-cache",
+ method: "upsertCachedIndexEntries",
+ key,
+ });
+
+ // If invalidating the negative marker failed, a surviving marker
+ // would falsely keep proving "nothing inserted since" — retry the DEL
+ // on its own. This is unconditional because deleting the marker is
+ // always safe (it only costs a DB read on the next lookup), and a
+ // connection failure can report the HSET as failed even when Redis
+ // applied it. A throw here is caught below and logged as a write
+ // failure.
+ if (results?.[2]?.[0]) {
+ await client.del(negKeyFor(key));
+ }
+
const hlen = results?.[3]?.[1];
if (typeof hlen === "number" && hlen > ENTRY_CAP) {
const raw = await client.hgetall(key);
diff --git a/apps/api/src/services/worker/nuq-reconciler-worker.ts b/apps/api/src/services/worker/nuq-reconciler-worker.ts
index 796ecb5f8..865da6549 100644
--- a/apps/api/src/services/worker/nuq-reconciler-worker.ts
+++ b/apps/api/src/services/worker/nuq-reconciler-worker.ts
@@ -4,6 +4,7 @@ import "../sentry";
import { setSentryServiceTag } from "../sentry";
import { logger as _logger } from "../../lib/logger";
import { reconcileConcurrencyQueue } from "../../lib/concurrency-queue-reconciler";
+import { repairCrawlJobDoneMarkers } from "../../lib/crawl-redis";
import { Counter, register } from "prom-client";
import Express from "express";
@@ -86,6 +87,12 @@ const reconcilerJobsRecoveredTotal = new Counter({
if (!reconcilerInFlight) {
reconcilerInFlight = true;
+ try {
+ await repairCrawlJobDoneMarkers(_logger);
+ } catch (error) {
+ _logger.error("Crawl completion marker repair run failed", { error });
+ }
+
try {
const summary = await reconcileConcurrencyQueue({
logger: _logger,
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index 323f6575b..87b838f69 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -18,6 +18,7 @@ import {
addCrawlJobs,
addCrawlJobDone,
crawlToCrawler,
+ queueCrawlJobDoneRepair,
recordRobotsBlocked,
recordThreatBlocked,
finishCrawlKickoff,
@@ -834,7 +835,22 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
await recordMonitorScrapeSuccess(job, doc);
logger.debug("Declaring job as done...");
- await addCrawlJobDone(job.data.crawl_id, job.id, true, logger);
+ try {
+ await addCrawlJobDone(job.data.crawl_id, job.id, true, logger);
+ } catch (e) {
+ // The scrape succeeded and its success webhook already went out —
+ // a bookkeeping failure must not route this job through the
+ // failure path (contradictory failure webhook, misrecorded job).
+ // Already logged canonically inside addCrawlJobDone.
+ logger.error("Failed to mark successful crawl job as done", {
+ crawlId: job.data.crawl_id,
+ jobId: job.id,
+ error: e,
+ });
+ // Durable fallback so the crawl's completion marker is retried by
+ // the reconciler instead of being lost for good.
+ await queueCrawlJobDoneRepair(job.data.crawl_id, job.id, true, logger);
+ }
} else {
try {
signal?.throwIfAborted();
@@ -944,7 +960,18 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
logger.debug("Declaring job as done...");
- await addCrawlJobDone(job.data.crawl_id, job.id, false, logger);
+ try {
+ await addCrawlJobDone(job.data.crawl_id, job.id, false, logger);
+ } catch (e) {
+ // Already logged canonically inside addCrawlJobDone; a throw here
+ // must not escape the error handler and fail the job a second time.
+ logger.error("Failed to declare failed crawl job as done", {
+ crawlId: job.data.crawl_id,
+ jobId: job.id,
+ error: e,
+ });
+ await queueCrawlJobDoneRepair(job.data.crawl_id, job.id, false, logger);
+ }
await redisEvictConnection.srem(
"crawl:" + job.data.crawl_id + ":visited_unique",
normalizeURL(job.data.url, sc),