Skip to content

Open source alternative to Semrush and Ahrefs

2.1 KB
Raw Blame
/**
 * KV-based live crawl progress.
 *
 * During a crawl, each crawled URL is appended to a KV key so the UI can
 * poll for a live feed of crawled pages (most recent first).
 *
 * The KV entry auto-expires after 30 minutes — it's only needed while
 * the audit is running. Once finalized, we explicitly delete it.
 */
import { env } from "cloudflare:workers";
import { z } from "zod";
import { jsonCodec } from "@/shared/json";

const KV_PREFIX = "audit-progress:";
const TTL_SECONDS = 30 * 60; // 30 minutes
const MAX_ENTRIES = 300;

const crawledUrlEntrySchema = z.object({
  url: z.string(),
  statusCode: z.number(),
  title: z.string(),
  /** Unix timestamp ms when this page was crawled */
  crawledAt: z.number(),
});

type CrawledUrlEntry = z.infer<typeof crawledUrlEntrySchema>;

const crawledEntriesCodec = jsonCodec(z.array(crawledUrlEntrySchema));

function parseCrawledEntries(json: string | null): CrawledUrlEntry[] {
  if (!json) return [];
  const parsed = crawledEntriesCodec.safeParse(json);
  return parsed.success ? parsed.data : [];
}

function key(auditId: string): string {
  return `${KV_PREFIX}${auditId}`;
}

/**
 * Append multiple crawled URL entries in one KV write.
 * New entries are prepended and the list is capped.
 */
async function pushCrawledUrls(
  auditId: string,
  nextEntries: CrawledUrlEntry[],
): Promise<void> {
  if (nextEntries.length === 0) return;

  const k = key(auditId);
  const existing = await env.KV.get(k, "text");
  const entries = parseCrawledEntries(existing);
  const merged = [...nextEntries, ...entries].slice(0, MAX_ENTRIES);

  await env.KV.put(k, JSON.stringify(merged), {
    expirationTtl: TTL_SECONDS,
  });
}

/**
 * Read all crawled URL entries for a running audit.
 * Returns newest-first.
 */
async function getCrawledUrls(auditId: string): Promise<CrawledUrlEntry[]> {
  const data = await env.KV.get(key(auditId), "text");
  return parseCrawledEntries(data);
}

/**
 * Delete the progress key (called after audit completes).
 */
async function clear(auditId: string): Promise<void> {
  await env.KV.delete(key(auditId));
}

export const AuditProgressKV = {
  pushCrawledUrls,
  getCrawledUrls,
  clear,
} as const;