Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

debug(crawl-redis): prove correct behaviour for ENG-3085

Gergő Móriczb75c3de8ec6b
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
index 595f4f065..c20275273 100644
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@ -2,7 +2,7 @@ import { InternalOptions } from "../scraper/scrapeURL";
import { ScrapeOptions, TeamFlags } from "../controllers/v1/types";
import { WebCrawler } from "../scraper/WebScraper/crawler";
import { redisEvictConnection } from "../services/redis";
-import { logger as _logger } from "./logger";
+import { logger as _logger, logger } from "./logger";
import { getAdjustedMaxDepth } from "../scraper/WebScraper/utils/maxDepthUtils";
import type { Logger } from "winston";
@@ -307,6 +307,21 @@ export async function lockURL(
"crawl:" + id + ":visited",
...permutations,
);
+ // TEMP: proving ENG-3085
+ if (x === permutations.length || x === 0) {
+ logger.debug("proof OK", {
+ proof: "ENG-3085",
+ permutations: permutations.length,
+ result: x,
+ });
+ } else {
+ logger.warn("proof failed", {
+ proof: "ENG-3085",
+ permutations: permutations.length,
+ result: x,
+ url,
+ });
+ }
res = x === permutations.length;
}