Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Preserve original URL case when storing search results
Developers Digestf84a2b0dfadf
diff --git a/apps/api/src/services/monitoring/search/run.test.ts b/apps/api/src/services/monitoring/search/run.test.tsindex 91839be11..398a6ddd0 100644--- a/apps/api/src/services/monitoring/search/run.test.ts+++ b/apps/api/src/services/monitoring/search/run.test.ts@@ -5,6 +5,7 @@ import {buildJudgePrompt,} from "./judge";import { canonicalizeUrl, stableSerpFingerprint } from "./dedupe";+import { hashMonitorUrl } from "../store";import { scrapeRequestSchema } from "../../../controllers/v2/types";// vi.mock is hoisted above declarations, so the mocks its factories reference@@ -133,6 +134,22 @@ describe("runSearchTarget orchestration", () => {expect(scrapePageMock).toHaveBeenCalledTimes(1);});+ it("preserves original URL case in the stored page (paths/query params are case-sensitive)", async () => {+ const mixedUrl = "https://example.com/watch?v=DAlfGz8tPc4";+ setSearchResults([+ { url: mixedUrl, title: "OpenAI reel", description: "clip" },+ ]);+ setVerdictsByUrl({ [mixedUrl]: verdict() });++ const out = await run();++ expect(out.pageUpserts[0].url).toBe(mixedUrl);+ expect(out.sources[0].url).toBe(mixedUrl);+ expect(out.pageUpserts[0].urlHash).toEqual(+ hashMonitorUrl(canonicalizeUrl(mixedUrl)),+ );+ });+it("skips scrape/judge for an already-seen unchanged page (same goalVersion)", async () => {setSearchResults([{diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.tsindex 080ca6d77..f5d7f1e35 100644--- a/apps/api/src/services/monitoring/search/run.ts+++ b/apps/api/src/services/monitoring/search/run.ts@@ -416,7 +416,7 @@ export async function runSearchTarget(params: {skipped += 1;sources.push({ url: c.url, title: c.title, status: "skipped" });pageUpserts.push({- url: canonical,+ url: c.url,urlHash: hashMonitorUrl(canonical),status: "skipped",scraped: false,@@ -442,7 +442,7 @@ export async function runSearchTarget(params: {: "watching";sources.push({ url: c.url, title: c.title, status: reusedStatus });pageUpserts.push({- url: canonical,+ url: c.url,urlHash: hashMonitorUrl(canonical),status: reusedStatus,metadata: {@@ -466,7 +466,7 @@ export async function runSearchTarget(params: {goalVersion,searchStatus: alreadyAlerted ? "already_seen" : "alert",eventKey: rawKey,- eventLabel: c.title || canonical,+ eventLabel: c.title || c.url,query: c.query,matchedQueries: c.matchedQueries,};@@ -478,7 +478,7 @@ export async function runSearchTarget(params: {eventKey: rawKey,});pageUpserts.push({- url: canonical,+ url: c.url,urlHash: hashMonitorUrl(canonical),status: "already_seen",scraped: false,@@ -497,7 +497,7 @@ export async function runSearchTarget(params: {} else {events.unshift({key: rawKey,- label: c.title || canonical,+ label: c.title || c.url,satisfiedAt: eventSatisfiedAt,alertCount: eventAlertCount,});@@ -509,7 +509,7 @@ export async function runSearchTarget(params: {eventKey: rawKey,});pageUpserts.push({- url: canonical,+ url: c.url,urlHash: hashMonitorUrl(canonical),status: "alert",scraped: false,@@ -609,7 +609,7 @@ export async function runSearchTarget(params: {...baseMeta,});pageUpserts.push({- url: canonical,+ url: c.url,urlHash: hashMonitorUrl(canonical),status: "ignored",scraped: depth === "deep",@@ -625,7 +625,7 @@ export async function runSearchTarget(params: {...baseMeta,});pageUpserts.push({- url: canonical,+ url: c.url,urlHash: hashMonitorUrl(canonical),status: "watching",scraped: depth === "deep",@@ -636,7 +636,7 @@ export async function runSearchTarget(params: {// Deterministic event key: dedup by canonical URL (no LLM event resolver).const eventKey = canonical;- const eventLabel = verdict.concept || canonical;+ const eventLabel = verdict.concept || c.url;// Re-alert on every new result only in every_new_result mode; otherwise dedup.const alreadySatisfied =@@ -652,7 +652,7 @@ export async function runSearchTarget(params: {...baseMeta,});pageUpserts.push({- url: canonical,+ url: c.url,urlHash: hashMonitorUrl(canonical),status: "already_seen",scraped: depth === "deep",@@ -686,7 +686,7 @@ export async function runSearchTarget(params: {...baseMeta,});pageUpserts.push({- url: canonical,+ url: c.url,urlHash: hashMonitorUrl(canonical),status: "alert",scraped: depth === "deep",