Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api/changeTracking): don't need DB row for getting the job content
Gergő Móricz536211f1fd3f
diff --git a/apps/api/src/scraper/scrapeURL/transformers/diff.ts b/apps/api/src/scraper/scrapeURL/transformers/diff.tsindex cba371d38..39561463a 100644--- a/apps/api/src/scraper/scrapeURL/transformers/diff.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/diff.ts@@ -1,11 +1,11 @@import { supabase_service } from "../../../services/supabase";import { Document } from "../../../controllers/v1/types";import { Meta } from "../index";-import { getJob } from "../../../controllers/v1/crawl-status";import gitDiff from "git-diff";import parseDiff from "parse-diff";import { generateCompletions } from "./llmExtract";import { hasFormatOfType } from "../../../lib/format-utils";+import { getJobFromGCS } from "../../../lib/gcs-jobs";async function extractDataWithSchema(content: string,@@ -113,11 +113,11 @@ export async function deriveDiff(| undefined| null = (res.data ?? [])[0] as any;- const job: {- returnvalue: Document;- } | null = data?.o_job_id ? await getJob(data.o_job_id) : null;- if (data && job && job?.returnvalue) {- const previousMarkdown = job.returnvalue.markdown!;+ const job: Document | null = data?.o_job_id+ ? ((await getJobFromGCS(data.o_job_id)?.[0]) ?? null)+ : null;+ if (data && job) {+ const previousMarkdown = job.markdown!;const currentMarkdown = document.markdown!;const transformer = (x: string) =>