The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Cache 404s in scrape-creators to avoid repeated failed requests (#3520)
Kiran K0446951d594e
diff --git a/apps/web/lib/api/scrape-creators/client.ts b/apps/web/lib/api/scrape-creators/client.tsindex 7f4820af4d..19d3a7aa30 100644--- a/apps/web/lib/api/scrape-creators/client.ts+++ b/apps/web/lib/api/scrape-creators/client.ts@@ -7,7 +7,7 @@ export const scrapeCreatorsFetch = createFetch({baseURL: "https://api.scrapecreators.com",retry: {type: "linear",- attempts: 2,+ attempts: 1,delay: 3000,},headers: {diff --git a/apps/web/lib/api/scrape-creators/get-social-content.ts b/apps/web/lib/api/scrape-creators/get-social-content.tsindex afbab85229..bd3a1255bb 100644--- a/apps/web/lib/api/scrape-creators/get-social-content.ts+++ b/apps/web/lib/api/scrape-creators/get-social-content.ts@@ -76,6 +76,17 @@ export async function getSocialContent({);if (error) {+ // Post not found+ // Cache empty result, so that we don't keep trying to scrape the same post.+ if (error.status === 404) {+ waitUntil(+ redis.set(cacheKey, EMPTY_SOCIAL_CONTENT, {+ ex: CACHE_TTL * 24 * 30,+ }),+ );+ }++ // We don't cache other errors because they are likely to be transient.return EMPTY_SOCIAL_CONTENT;}