Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

feat: IndexNow

Blue Mousef98490c8e1f3
diff --git a/backend/.env.example b/backend/.env.example
index 3658a7483..6722b2bc8 100644
--- a/backend/.env.example
+++ b/backend/.env.example
@@ -154,6 +154,10 @@ [email protected]
# Delay before the live checkout pulls the new blog commit from GitHub.
RANKPINE_BLOG_REFRESH_DELAY_MS=30000
+# IndexNow URL submission (Cloud edition only)
+# Use the same key in the web service so /{key}.txt can verify ownership.
+INDEXNOW_KEY=
+
# Managed reverse proxy ("Bypass adblockers") edge auth
# Shared secret the proxy.swetrix.org edge box sends in `X-Edge-Api-Key` to
# call the /v1/proxy-domain/allow and /v1/proxy-domain/active endpoints.
diff --git a/backend/apps/cloud/src/webhook/indexnow.service.spec.ts b/backend/apps/cloud/src/webhook/indexnow.service.spec.ts
new file mode 100644
index 000000000..c4adb88d2
--- /dev/null
+++ b/backend/apps/cloud/src/webhook/indexnow.service.spec.ts
@@ -0,0 +1,83 @@
+import { AppLoggerService } from '../logger/logger.service'
+import { IndexNowService } from './indexnow.service'
+
+const ORIGINAL_KEY = process.env.INDEXNOW_KEY
+
+describe('IndexNowService', () => {
+ const logger = {
+ log: jest.fn(),
+ warn: jest.fn(),
+ } as unknown as AppLoggerService
+ const service = new IndexNowService(logger)
+
+ beforeEach(() => {
+ process.env.INDEXNOW_KEY = 'indexnow-test-key'
+ jest.clearAllMocks()
+ })
+
+ afterAll(() => {
+ if (ORIGINAL_KEY === undefined) {
+ delete process.env.INDEXNOW_KEY
+ } else {
+ process.env.INDEXNOW_KEY = ORIGINAL_KEY
+ }
+ })
+
+ it('submits unique URLs with the root key location', async () => {
+ const fetchMock = jest.spyOn(global, 'fetch').mockResolvedValue(
+ new Response(null, {
+ status: 202,
+ }),
+ )
+
+ await expect(
+ service.submit([
+ 'https://swetrix.com/blog/new-post',
+ 'https://swetrix.com/blog/new-post',
+ 'https://swetrix.com/blog/updated-post',
+ ]),
+ ).resolves.toBe(true)
+
+ expect(fetchMock).toHaveBeenCalledWith(
+ 'https://api.indexnow.org/indexnow',
+ expect.objectContaining({
+ method: 'POST',
+ headers: {
+ 'Content-Type': 'application/json; charset=utf-8',
+ },
+ body: JSON.stringify({
+ host: 'swetrix.com',
+ key: 'indexnow-test-key',
+ keyLocation: 'https://swetrix.com/indexnow-test-key.txt',
+ urlList: [
+ 'https://swetrix.com/blog/new-post',
+ 'https://swetrix.com/blog/updated-post',
+ ],
+ }),
+ }),
+ )
+
+ fetchMock.mockRestore()
+ })
+
+ it('does nothing when the key is not configured', async () => {
+ delete process.env.INDEXNOW_KEY
+ const fetchMock = jest.spyOn(global, 'fetch')
+
+ await expect(
+ service.submit(['https://swetrix.com/blog/new-post']),
+ ).resolves.toBe(false)
+ expect(fetchMock).not.toHaveBeenCalled()
+
+ fetchMock.mockRestore()
+ })
+
+ it('rejects URLs from different hosts', async () => {
+ await expect(
+ service.submit([
+ 'https://swetrix.com/blog/new-post',
+ 'https://example.com/blog/new-post',
+ ]),
+ ).rejects.toThrow('IndexNow URLs must use HTTP(S) and share one host')
+ })
+})
diff --git a/backend/apps/cloud/src/webhook/indexnow.service.ts b/backend/apps/cloud/src/webhook/indexnow.service.ts
new file mode 100644
index 000000000..b579b5e7c
--- /dev/null
+++ b/backend/apps/cloud/src/webhook/indexnow.service.ts
@@ -0,0 +1,93 @@
+import { Injectable } from '@nestjs/common'
+
+import { AppLoggerService } from '../logger/logger.service'
+
+const INDEXNOW_ENDPOINT = 'https://api.indexnow.org/indexnow'
+const INDEXNOW_KEY_PATTERN = /^[A-Za-z0-9-]{8,128}$/
+const MAX_URLS_PER_REQUEST = 10_000
+
+@Injectable()
+export class IndexNowService {
+ constructor(private readonly logger: AppLoggerService) {}
+
+ async submit(urls: string[]): Promise<boolean> {
+ const key = process.env.INDEXNOW_KEY?.trim()
+
+ if (!key) {
+ this.logger.warn(
+ 'IndexNow URL submission is disabled because INDEXNOW_KEY is missing',
+ 'IndexNow',
+ true,
+ )
+ return false
+ }
+
+ if (!INDEXNOW_KEY_PATTERN.test(key)) {
+ throw new Error(
+ 'INDEXNOW_KEY must be 8 to 128 letters, numbers, or dashes',
+ )
+ }
+
+ const uniqueUrls = [...new Set(urls)]
+
+ if (uniqueUrls.length === 0) {
+ return false
+ }
+
+ if (uniqueUrls.length > MAX_URLS_PER_REQUEST) {
+ throw new Error(
+ `IndexNow accepts at most ${MAX_URLS_PER_REQUEST} URLs per request`,
+ )
+ }
+
+ const parsedUrls = uniqueUrls.map((url) => new URL(url))
+ const host = parsedUrls[0].host
+
+ if (
+ parsedUrls.some(
+ (url) =>
+ !['http:', 'https:'].includes(url.protocol) || url.host !== host,
+ )
+ ) {
+ throw new Error('IndexNow URLs must use HTTP(S) and share one host')
+ }
+
+ const keyLocation = new URL(`/${key}.txt`, parsedUrls[0].origin).toString()
+ let response: Response
+
+ try {
+ response = await fetch(INDEXNOW_ENDPOINT, {
+ method: 'POST',
+ signal: AbortSignal.timeout(10_000),
+ headers: {
+ 'Content-Type': 'application/json; charset=utf-8',
+ },
+ body: JSON.stringify({
+ host,
+ key,
+ keyLocation,
+ urlList: uniqueUrls,
+ }),
+ })
+ } catch (error) {
+ const reason = error instanceof Error ? error.message : String(error)
+ throw new Error(`IndexNow request failed: ${reason}`)
+ }
+
+ if (!response.ok) {
+ throw new Error(`IndexNow returned HTTP ${response.status}`)
+ }
+
+ this.logger.log(
+ {
+ status: response.status,
+ urlCount: uniqueUrls.length,
+ urls: uniqueUrls,
+ },
+ 'IndexNow',
+ true,
+ )
+
+ return true
+ }
+}
diff --git a/backend/apps/cloud/src/webhook/rankpine-blog.service.spec.ts b/backend/apps/cloud/src/webhook/rankpine-blog.service.spec.ts
index 602bc9380..973b1710a 100644
--- a/backend/apps/cloud/src/webhook/rankpine-blog.service.spec.ts
+++ b/backend/apps/cloud/src/webhook/rankpine-blog.service.spec.ts
@@ -3,6 +3,7 @@ import { UnauthorizedException } from '@nestjs/common'
import { AppLoggerService } from '../logger/logger.service'
import { BlogService } from '../blog/blog.service'
+import { IndexNowService } from './indexnow.service'
import { RankPineBlogService } from './rankpine-blog.service'
const SECRET = 'rankpine-test-secret'
@@ -43,7 +44,10 @@ describe('RankPineBlogService', () => {
const blogService = {
clearSitemapCache: jest.fn(),
} as unknown as BlogService
- const service = new RankPineBlogService(logger, blogService)
+ const indexNowService = {
+ submit: jest.fn(),
+ } as unknown as IndexNowService
+ const service = new RankPineBlogService(logger, blogService, indexNowService)
beforeEach(() => {
process.env.RANKPINE_BLOG_WEBHOOK_SECRET = SECRET
@@ -101,7 +105,7 @@ describe('RankPineBlogService', () => {
const refreshSpy = jest
.spyOn(
service as unknown as {
- refreshLocalBlogPosts: () => Promise<void>
+ refreshLocalBlogPosts: (urls: string[]) => Promise<void>
},
'refreshLocalBlogPosts',
)
@@ -239,7 +243,9 @@ describe('RankPineBlogService', () => {
jest.advanceTimersByTime(29_999)
expect(refreshSpy).not.toHaveBeenCalled()
jest.advanceTimersByTime(1)
- expect(refreshSpy).toHaveBeenCalledTimes(1)
+ expect(refreshSpy).toHaveBeenCalledWith([
+ 'https://swetrix.com/blog/automated-post',
+ ])
refreshSpy.mockRestore()
fetchMock.mockRestore()
diff --git a/backend/apps/cloud/src/webhook/rankpine-blog.service.ts b/backend/apps/cloud/src/webhook/rankpine-blog.service.ts
index 3d13b6997..877e89b23 100644
--- a/backend/apps/cloud/src/webhook/rankpine-blog.service.ts
+++ b/backend/apps/cloud/src/webhook/rankpine-blog.service.ts
@@ -13,6 +13,7 @@ import { z } from 'zod'
import { AppLoggerService } from '../logger/logger.service'
import { BlogService } from '../blog/blog.service'
+import { IndexNowService } from './indexnow.service'
const GITHUB_API_URL = 'https://api.github.com'
const GITHUB_API_VERSION = '2026-03-10'
@@ -257,10 +258,12 @@ function articleMarkdown(
@Injectable()
export class RankPineBlogService {
private refreshTimer: NodeJS.Timeout | null = null
+ private pendingIndexNowUrls = new Set<string>()
constructor(
private readonly logger: AppLoggerService,
private readonly blogService: BlogService,
+ private readonly indexNowService: IndexNowService,
) {}
private getConfig(): PublisherConfig {
@@ -302,19 +305,23 @@ export class RankPineBlogService {
}
}
- private scheduleLocalRefresh(delayMs: number): void {
+ private scheduleLocalRefresh(delayMs: number, url: string): void {
+ this.pendingIndexNowUrls.add(url)
+
if (this.refreshTimer) {
clearTimeout(this.refreshTimer)
}
this.refreshTimer = setTimeout(() => {
this.refreshTimer = null
- void this.refreshLocalBlogPosts()
+ const urls = [...this.pendingIndexNowUrls]
+ this.pendingIndexNowUrls.clear()
+ void this.refreshLocalBlogPosts(urls)
}, delayMs)
this.refreshTimer.unref()
}
- private async refreshLocalBlogPosts(): Promise<void> {
+ private async refreshLocalBlogPosts(urls: string[]): Promise<void> {
const cwd = backendRoot()
try {
@@ -345,6 +352,20 @@ export class RankPineBlogService {
'RankPine blog submodule refresh',
true,
)
+ return
+ }
+
+ try {
+ await this.indexNowService.submit(urls)
+ } catch (error) {
+ this.logger.error(
+ {
+ reason: error instanceof Error ? error.message : String(error),
+ urls,
+ },
+ 'IndexNow URL submission',
+ true,
+ )
}
}
@@ -556,12 +577,16 @@ export class RankPineBlogService {
'POST /webhook/rankpine',
true,
)
- this.scheduleLocalRefresh(config.refreshDelayMs)
+ const publicUrl = new URL(`/blog/${slug}`, expectedOrigin).toString()
+
+ if (result.changed) {
+ this.scheduleLocalRefresh(config.refreshDelayMs, publicUrl)
+ }
return {
ok: true,
id: path,
- url: new URL(`/blog/${slug}`, expectedOrigin).toString(),
+ url: publicUrl,
changed: result.changed,
}
}
diff --git a/backend/apps/cloud/src/webhook/webhook.module.ts b/backend/apps/cloud/src/webhook/webhook.module.ts
index cf2d5879b..d910624dd 100644
--- a/backend/apps/cloud/src/webhook/webhook.module.ts
+++ b/backend/apps/cloud/src/webhook/webhook.module.ts
@@ -8,6 +8,7 @@ import { MailerModule } from '../mailer/mailer.module'
import { WebhookService } from './webhook.service'
import { RevenueModule } from '../revenue/revenue.module'
import { BlogModule } from '../blog/blog.module'
+import { IndexNowService } from './indexnow.service'
import { RankPineBlogService } from './rankpine-blog.service'
@Module({
@@ -19,7 +20,7 @@ import { RankPineBlogService } from './rankpine-blog.service'
RevenueModule,
BlogModule,
],
- providers: [WebhookService, RankPineBlogService],
+ providers: [WebhookService, IndexNowService, RankPineBlogService],
exports: [WebhookService],
controllers: [WebhookController],
})
diff --git a/web/.env.example b/web/.env.example
index cf532776b..549be32b5 100644
--- a/web/.env.example
+++ b/web/.env.example
@@ -5,6 +5,9 @@ COOKIE_DOMAIN=swetrix.com
DISABLE_MARKETING_PAGES=
PADDLE_CLIENT_SIDE_TOKEN=
+# Use the same IndexNow key as the backend service.
+INDEXNOW_KEY=
+
PORT=3000
# Community Edition (Selfhosted) configuration
diff --git a/web/app/entry.server.tsx b/web/app/entry.server.tsx
index 990a6e5dc..0112b282c 100644
--- a/web/app/entry.server.tsx
+++ b/web/app/entry.server.tsx
@@ -13,6 +13,7 @@ import { isSelfhosted, MAIN_URL } from './lib/constants'
// Reject/cancel all pending promises after 5 seconds
const streamTimeout = 5000
+const INDEXNOW_KEY_PATTERN = /^[A-Za-z0-9-]{8,128}$/
const { sitemap } = createSitemapGenerator({
siteUrl: MAIN_URL,
@@ -47,6 +48,20 @@ export default async function handleRequest(
routerContext: RouterContextProvider,
) {
const url = new URL(request.url)
+ const indexNowKey = process.env.INDEXNOW_KEY?.trim()
+
+ if (
+ indexNowKey &&
+ INDEXNOW_KEY_PATTERN.test(indexNowKey) &&
+ url.pathname === `/${indexNowKey}.txt`
+ ) {
+ return new Response(indexNowKey, {
+ headers: {
+ 'Content-Type': 'text/plain; charset=utf-8',
+ 'Cache-Control': 'public, max-age=3600',
+ },
+ })
+ }
if (!isSelfhosted && url.pathname === '/sitemap.xml') {
return getSitemapIndexResponse()