Open source alternative to Semrush and Ahrefs
TypeScript backlink-analysisgoogle-search-console-mcpkeyword-researchmcpseoseo-agentseo-auditseo-automationseo-skillsseo-toolssite-audit
/**
* Shopify crawler access.
*
* Shopify storefronts rate-limit (429) and sometimes block crawlers they
* haven't authorized. The merchant mints a domain-scoped signature in Shopify
* admin (Online Store -> Preferences -> Crawler access) and the crawler
* replays it as three static headers on every request to that domain.
*
* These helpers are pure so both the crawler and the input validation can use
* them.
*/
/** Constant Shopify expects, quotes included. We supply it; users never see it. */
export const SHOPIFY_SIGNATURE_AGENT = '"https://shopify.com"';
export const SHOPIFY_CRAWLER_ACCESS_DOC_URL =
"https://help.shopify.com/en/manual/promoting-marketing/seo/crawling-your-store";
export const MAX_SIGNATURE_VALUE_LENGTH = 4096;
/**
* Credential headers bound to the host they were issued for. The binding is
* the security boundary: these values are bot-protection credentials and must
* never be sent to another site, including across a redirect hop.
*/
export interface CrawlerAccess {
host: string;
headers: Record<string, string>;
/** Checked on every request: a long crawl can outlive the signature. */
expiresAt: string | null;
}
export function shopifyCrawlerHeaders(
signatureInput: string,
signature: string,
): Record<string, string> {
return {
"Signature-Input": signatureInput,
Signature: signature,
"Signature-Agent": SHOPIFY_SIGNATURE_AGENT,
};
}
/**
* Normalize user input into a bare lowercase hostname. Accepts a full URL
* (people paste one) as well as a bare host. Null = not a usable hostname.
*/
export function normalizeCrawlerHost(input: string): string | null {
const raw = input.trim().toLowerCase();
if (!raw) return null;
let url: URL;
try {
url = new URL(raw.includes("://") ? raw : `https://${raw}`);
} catch {
return null;
}
// Other schemes parse hosts differently ("h://5.6" keeps "5.6", which https
// reads as 5.0.0.6), so the saved host would never match a request.
if (url.protocol !== "https:" && url.protocol !== "http:") return null;
const host = url.hostname.replace(/\.$/, "");
if (!/^[a-z0-9-]+(\.[a-z0-9-]+)+$/.test(host)) return null;
return host;
}
/**
* Headers to send with a request to `url`, or `{}` when it isn't the host the
* signature was created for, over https, before its expiry. The host match is
* exact: Shopify scopes a signature to one domain, so `www.store.com` and
* `store.com` need their own. A redirect to plain http must not replay the
* signature in cleartext.
*/
export function crawlerHeadersFor(
url: string,
access: CrawlerAccess | null | undefined,
): Record<string, string> {
if (!access || isCrawlerAccessExpired(access.expiresAt)) return {};
let parsed: URL;
try {
parsed = new URL(url);
} catch {
return {};
}
return parsed.protocol === "https:" &&
parsed.hostname.toLowerCase() === access.host
? access.headers
: {};
}
/**
* Shopify's `Signature-Input` is an RFC 9421 signature parameter list that may
* carry `;expires=<unix seconds>`. Nothing beyond that parameter is assumed.
*/
export function parseSignatureExpiry(signatureInput: string): string | null {
const match = signatureInput.match(/expires=(\d+)/);
if (!match) return null;
const expiresAt = new Date(Number(match[1]) * 1000);
return Number.isNaN(expiresAt.getTime()) ? null : expiresAt.toISOString();
}
/** Shopify rejects a signature past its `expires`, so it is never replayed. */
export function isCrawlerAccessExpired(
expiresAt: string | null,
now = Date.now(),
): boolean {
return expiresAt !== null && new Date(expiresAt).getTime() <= now;
}