📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
feat: IndexNow
Blue Mousef98490c8e1f3
diff --git a/backend/.env.example b/backend/.env.exampleindex 3658a7483..6722b2bc8 100644--- a/backend/.env.example+++ b/backend/.env.example@@ -154,6 +154,10 @@ [email protected]# Delay before the live checkout pulls the new blog commit from GitHub.RANKPINE_BLOG_REFRESH_DELAY_MS=30000+# IndexNow URL submission (Cloud edition only)+# Use the same key in the web service so /{key}.txt can verify ownership.+INDEXNOW_KEY=+# Managed reverse proxy ("Bypass adblockers") edge auth# Shared secret the proxy.swetrix.org edge box sends in `X-Edge-Api-Key` to# call the /v1/proxy-domain/allow and /v1/proxy-domain/active endpoints.diff --git a/backend/apps/cloud/src/webhook/indexnow.service.spec.ts b/backend/apps/cloud/src/webhook/indexnow.service.spec.tsnew file mode 100644index 000000000..c4adb88d2--- /dev/null+++ b/backend/apps/cloud/src/webhook/indexnow.service.spec.ts@@ -0,0 +1,83 @@+import { AppLoggerService } from '../logger/logger.service'+import { IndexNowService } from './indexnow.service'++const ORIGINAL_KEY = process.env.INDEXNOW_KEY++describe('IndexNowService', () => {+ const logger = {+ log: jest.fn(),+ warn: jest.fn(),+ } as unknown as AppLoggerService+ const service = new IndexNowService(logger)++ beforeEach(() => {+ process.env.INDEXNOW_KEY = 'indexnow-test-key'+ jest.clearAllMocks()+ })++ afterAll(() => {+ if (ORIGINAL_KEY === undefined) {+ delete process.env.INDEXNOW_KEY+ } else {+ process.env.INDEXNOW_KEY = ORIGINAL_KEY+ }+ })++ it('submits unique URLs with the root key location', async () => {+ const fetchMock = jest.spyOn(global, 'fetch').mockResolvedValue(+ new Response(null, {+ status: 202,+ }),+ )++ await expect(+ service.submit([+ 'https://swetrix.com/blog/new-post',+ 'https://swetrix.com/blog/new-post',+ 'https://swetrix.com/blog/updated-post',+ ]),+ ).resolves.toBe(true)++ expect(fetchMock).toHaveBeenCalledWith(+ 'https://api.indexnow.org/indexnow',+ expect.objectContaining({+ method: 'POST',+ headers: {+ 'Content-Type': 'application/json; charset=utf-8',+ },+ body: JSON.stringify({+ host: 'swetrix.com',+ key: 'indexnow-test-key',+ keyLocation: 'https://swetrix.com/indexnow-test-key.txt',+ urlList: [+ 'https://swetrix.com/blog/new-post',+ 'https://swetrix.com/blog/updated-post',+ ],+ }),+ }),+ )++ fetchMock.mockRestore()+ })++ it('does nothing when the key is not configured', async () => {+ delete process.env.INDEXNOW_KEY+ const fetchMock = jest.spyOn(global, 'fetch')++ await expect(+ service.submit(['https://swetrix.com/blog/new-post']),+ ).resolves.toBe(false)+ expect(fetchMock).not.toHaveBeenCalled()++ fetchMock.mockRestore()+ })++ it('rejects URLs from different hosts', async () => {+ await expect(+ service.submit([+ 'https://swetrix.com/blog/new-post',+ 'https://example.com/blog/new-post',+ ]),+ ).rejects.toThrow('IndexNow URLs must use HTTP(S) and share one host')+ })+})diff --git a/backend/apps/cloud/src/webhook/indexnow.service.ts b/backend/apps/cloud/src/webhook/indexnow.service.tsnew file mode 100644index 000000000..b579b5e7c--- /dev/null+++ b/backend/apps/cloud/src/webhook/indexnow.service.ts@@ -0,0 +1,93 @@+import { Injectable } from '@nestjs/common'++import { AppLoggerService } from '../logger/logger.service'++const INDEXNOW_ENDPOINT = 'https://api.indexnow.org/indexnow'+const INDEXNOW_KEY_PATTERN = /^[A-Za-z0-9-]{8,128}$/+const MAX_URLS_PER_REQUEST = 10_000++@Injectable()+export class IndexNowService {+ constructor(private readonly logger: AppLoggerService) {}++ async submit(urls: string[]): Promise<boolean> {+ const key = process.env.INDEXNOW_KEY?.trim()++ if (!key) {+ this.logger.warn(+ 'IndexNow URL submission is disabled because INDEXNOW_KEY is missing',+ 'IndexNow',+ true,+ )+ return false+ }++ if (!INDEXNOW_KEY_PATTERN.test(key)) {+ throw new Error(+ 'INDEXNOW_KEY must be 8 to 128 letters, numbers, or dashes',+ )+ }++ const uniqueUrls = [...new Set(urls)]++ if (uniqueUrls.length === 0) {+ return false+ }++ if (uniqueUrls.length > MAX_URLS_PER_REQUEST) {+ throw new Error(+ `IndexNow accepts at most ${MAX_URLS_PER_REQUEST} URLs per request`,+ )+ }++ const parsedUrls = uniqueUrls.map((url) => new URL(url))+ const host = parsedUrls[0].host++ if (+ parsedUrls.some(+ (url) =>+ !['http:', 'https:'].includes(url.protocol) || url.host !== host,+ )+ ) {+ throw new Error('IndexNow URLs must use HTTP(S) and share one host')+ }++ const keyLocation = new URL(`/${key}.txt`, parsedUrls[0].origin).toString()+ let response: Response++ try {+ response = await fetch(INDEXNOW_ENDPOINT, {+ method: 'POST',+ signal: AbortSignal.timeout(10_000),+ headers: {+ 'Content-Type': 'application/json; charset=utf-8',+ },+ body: JSON.stringify({+ host,+ key,+ keyLocation,+ urlList: uniqueUrls,+ }),+ })+ } catch (error) {+ const reason = error instanceof Error ? error.message : String(error)+ throw new Error(`IndexNow request failed: ${reason}`)+ }++ if (!response.ok) {+ throw new Error(`IndexNow returned HTTP ${response.status}`)+ }++ this.logger.log(+ {+ status: response.status,+ urlCount: uniqueUrls.length,+ urls: uniqueUrls,+ },+ 'IndexNow',+ true,+ )++ return true+ }+}diff --git a/backend/apps/cloud/src/webhook/rankpine-blog.service.spec.ts b/backend/apps/cloud/src/webhook/rankpine-blog.service.spec.tsindex 602bc9380..973b1710a 100644--- a/backend/apps/cloud/src/webhook/rankpine-blog.service.spec.ts+++ b/backend/apps/cloud/src/webhook/rankpine-blog.service.spec.ts@@ -3,6 +3,7 @@ import { UnauthorizedException } from '@nestjs/common'import { AppLoggerService } from '../logger/logger.service'import { BlogService } from '../blog/blog.service'+import { IndexNowService } from './indexnow.service'import { RankPineBlogService } from './rankpine-blog.service'const SECRET = 'rankpine-test-secret'@@ -43,7 +44,10 @@ describe('RankPineBlogService', () => {const blogService = {clearSitemapCache: jest.fn(),} as unknown as BlogService- const service = new RankPineBlogService(logger, blogService)+ const indexNowService = {+ submit: jest.fn(),+ } as unknown as IndexNowService+ const service = new RankPineBlogService(logger, blogService, indexNowService)beforeEach(() => {process.env.RANKPINE_BLOG_WEBHOOK_SECRET = SECRET@@ -101,7 +105,7 @@ describe('RankPineBlogService', () => {const refreshSpy = jest.spyOn(service as unknown as {- refreshLocalBlogPosts: () => Promise<void>+ refreshLocalBlogPosts: (urls: string[]) => Promise<void>},'refreshLocalBlogPosts',)@@ -239,7 +243,9 @@ describe('RankPineBlogService', () => {jest.advanceTimersByTime(29_999)expect(refreshSpy).not.toHaveBeenCalled()jest.advanceTimersByTime(1)- expect(refreshSpy).toHaveBeenCalledTimes(1)+ expect(refreshSpy).toHaveBeenCalledWith([+ 'https://swetrix.com/blog/automated-post',+ ])refreshSpy.mockRestore()fetchMock.mockRestore()diff --git a/backend/apps/cloud/src/webhook/rankpine-blog.service.ts b/backend/apps/cloud/src/webhook/rankpine-blog.service.tsindex 3d13b6997..877e89b23 100644--- a/backend/apps/cloud/src/webhook/rankpine-blog.service.ts+++ b/backend/apps/cloud/src/webhook/rankpine-blog.service.ts@@ -13,6 +13,7 @@ import { z } from 'zod'import { AppLoggerService } from '../logger/logger.service'import { BlogService } from '../blog/blog.service'+import { IndexNowService } from './indexnow.service'const GITHUB_API_URL = 'https://api.github.com'const GITHUB_API_VERSION = '2026-03-10'@@ -257,10 +258,12 @@ function articleMarkdown(@Injectable()export class RankPineBlogService {private refreshTimer: NodeJS.Timeout | null = null+ private pendingIndexNowUrls = new Set<string>()constructor(private readonly logger: AppLoggerService,private readonly blogService: BlogService,+ private readonly indexNowService: IndexNowService,) {}private getConfig(): PublisherConfig {@@ -302,19 +305,23 @@ export class RankPineBlogService {}}- private scheduleLocalRefresh(delayMs: number): void {+ private scheduleLocalRefresh(delayMs: number, url: string): void {+ this.pendingIndexNowUrls.add(url)+if (this.refreshTimer) {clearTimeout(this.refreshTimer)}this.refreshTimer = setTimeout(() => {this.refreshTimer = null- void this.refreshLocalBlogPosts()+ const urls = [...this.pendingIndexNowUrls]+ this.pendingIndexNowUrls.clear()+ void this.refreshLocalBlogPosts(urls)}, delayMs)this.refreshTimer.unref()}- private async refreshLocalBlogPosts(): Promise<void> {+ private async refreshLocalBlogPosts(urls: string[]): Promise<void> {const cwd = backendRoot()try {@@ -345,6 +352,20 @@ export class RankPineBlogService {'RankPine blog submodule refresh',true,)+ return+ }++ try {+ await this.indexNowService.submit(urls)+ } catch (error) {+ this.logger.error(+ {+ reason: error instanceof Error ? error.message : String(error),+ urls,+ },+ 'IndexNow URL submission',+ true,+ )}}@@ -556,12 +577,16 @@ export class RankPineBlogService {'POST /webhook/rankpine',true,)- this.scheduleLocalRefresh(config.refreshDelayMs)+ const publicUrl = new URL(`/blog/${slug}`, expectedOrigin).toString()++ if (result.changed) {+ this.scheduleLocalRefresh(config.refreshDelayMs, publicUrl)+ }return {ok: true,id: path,- url: new URL(`/blog/${slug}`, expectedOrigin).toString(),+ url: publicUrl,changed: result.changed,}}diff --git a/backend/apps/cloud/src/webhook/webhook.module.ts b/backend/apps/cloud/src/webhook/webhook.module.tsindex cf2d5879b..d910624dd 100644--- a/backend/apps/cloud/src/webhook/webhook.module.ts+++ b/backend/apps/cloud/src/webhook/webhook.module.ts@@ -8,6 +8,7 @@ import { MailerModule } from '../mailer/mailer.module'import { WebhookService } from './webhook.service'import { RevenueModule } from '../revenue/revenue.module'import { BlogModule } from '../blog/blog.module'+import { IndexNowService } from './indexnow.service'import { RankPineBlogService } from './rankpine-blog.service'@Module({@@ -19,7 +20,7 @@ import { RankPineBlogService } from './rankpine-blog.service'RevenueModule,BlogModule,],- providers: [WebhookService, RankPineBlogService],+ providers: [WebhookService, IndexNowService, RankPineBlogService],exports: [WebhookService],controllers: [WebhookController],})diff --git a/web/.env.example b/web/.env.exampleindex cf532776b..549be32b5 100644--- a/web/.env.example+++ b/web/.env.example@@ -5,6 +5,9 @@ COOKIE_DOMAIN=swetrix.comDISABLE_MARKETING_PAGES=PADDLE_CLIENT_SIDE_TOKEN=+# Use the same IndexNow key as the backend service.+INDEXNOW_KEY=+PORT=3000# Community Edition (Selfhosted) configurationdiff --git a/web/app/entry.server.tsx b/web/app/entry.server.tsxindex 990a6e5dc..0112b282c 100644--- a/web/app/entry.server.tsx+++ b/web/app/entry.server.tsx@@ -13,6 +13,7 @@ import { isSelfhosted, MAIN_URL } from './lib/constants'// Reject/cancel all pending promises after 5 secondsconst streamTimeout = 5000+const INDEXNOW_KEY_PATTERN = /^[A-Za-z0-9-]{8,128}$/const { sitemap } = createSitemapGenerator({siteUrl: MAIN_URL,@@ -47,6 +48,20 @@ export default async function handleRequest(routerContext: RouterContextProvider,) {const url = new URL(request.url)+ const indexNowKey = process.env.INDEXNOW_KEY?.trim()++ if (+ indexNowKey &&+ INDEXNOW_KEY_PATTERN.test(indexNowKey) &&+ url.pathname === `/${indexNowKey}.txt`+ ) {+ return new Response(indexNowKey, {+ headers: {+ 'Content-Type': 'text/plain; charset=utf-8',+ 'Cache-Control': 'public, max-age=3600',+ },+ })+ }if (!isSelfhosted && url.pathname === '/sitemap.xml') {return getSitemapIndexResponse()