Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.
TypeScript aeoai-agentai-searchanswer-engine-optimizationbrand-monitoringchatgptclaudecopilotdockergeminigeogoogle-ai-overviewsgrokmcpnextjsopen-sourceperplexityseosupabasetypescript
fix: preserve YouTube video IDs in citation URLs (#558)
Abdullah4aeec1bc8a66
diff --git a/web/src/lib/actions/citations.ts b/web/src/lib/actions/citations.tsindex e402f45dc..2e243cdcf 100644--- a/web/src/lib/actions/citations.ts+++ b/web/src/lib/actions/citations.ts@@ -11,6 +11,7 @@ import {SOURCE_CATEGORIES,} from '@/lib/citations/classify';import { classifyArticleType, type ArticleType } from '@/lib/citations/article-type';+import { citationUrlMatchKey, normalizeCitationUrl } from '@/lib/citations/normalize';// ─── Types ────────────────────────────────────────────────────────────────────@@ -78,40 +79,6 @@ export interface CitationsOverview {// ─── Helpers ──────────────────────────────────────────────────────────────────-/**- * Canonical URL key for citation aggregation: strip query/fragment and one- * trailing slash. The overview tables, the prompt detail Top Sources card and- * the per-URL detail page (#535) must all bucket with this exact rule so- * their counts agree for the same URL.- */-function normalizeCitationUrl(raw: string): string {- try {- const parsed = new URL(raw);- parsed.search = '';- parsed.hash = '';- return parsed.toString().replace(/\/$/, '');- } catch {- return raw;- }-}--/**- * Looser cross-source match key (host without `www.` + path without trailing- * slashes) — mirrors `urlMatchKey` in prompt-workflow.ts / the server's- * `normalizeUrlForMatch`, so the owned-URL bridges match targeted URLs and- * traffic-log URLs the same way the cited-stats pipeline does.- */-function citationUrlMatchKey(raw: string): string | null {- try {- const parsed = new URL(raw.trim());- const host = parsed.hostname.toLowerCase().replace(/^www\./, '');- const path = parsed.pathname.replace(/\/+$/, '');- return `${host}${path}`;- } catch {- return null;- }-}-function resolveDateRange(filters: CitationsFilters): { from?: string; to?: string } {if (filters.datePreset === 'custom') {return { from: filters.dateFrom, to: filters.dateTo };@@ -356,7 +323,7 @@ export async function getCitationsOverview(}domainMap.set(host, existingDomain);- // URL aggregation (strip query/fragment and trailing slash for dedupe).+ // URL aggregation (preserve query parameters that identify the page).const normalizedUrl = normalizeCitationUrl(cite.url);const existingUrl = urlMap.get(normalizedUrl) ?? {url: normalizedUrl,diff --git a/web/src/lib/actions/tracking.ts b/web/src/lib/actions/tracking.tsindex 0a6e02d16..afa002f6b 100644--- a/web/src/lib/actions/tracking.ts+++ b/web/src/lib/actions/tracking.ts@@ -18,6 +18,7 @@ import {normalizeDomain,type SourceCategory,} from '@/lib/citations/classify';+import { normalizeCitationUrl } from '@/lib/citations/normalize';import { getTopicById } from '@/lib/actions/topic';import { getOrgPlan } from '@/lib/guards/plan-guard';import { getPromptVolumes } from '@/lib/actions/volumes';@@ -940,17 +941,8 @@ export async function getPromptDetail(if (modelKey) agg.models.add(modelKey);sourceMap.set(host, agg);- // URL aggregation (strip query/fragment and trailing slash for dedupe,- // same as getCitationsOverview).- let normalizedUrl = cite.url;- try {- const parsed = new URL(cite.url);- parsed.search = '';- parsed.hash = '';- normalizedUrl = parsed.toString().replace(/\/$/, '');- } catch {- // leave as-is- }+ // URL aggregation uses the same host-aware rule as getCitationsOverview.+ const normalizedUrl = normalizeCitationUrl(cite.url);const urlAgg = urlMap.get(normalizedUrl) ?? {url: normalizedUrl,domain: host,diff --git a/web/src/lib/citations/normalize.test.ts b/web/src/lib/citations/normalize.test.tsnew file mode 100644index 000000000..653223dbb--- /dev/null+++ b/web/src/lib/citations/normalize.test.ts@@ -0,0 +1,55 @@+import { describe, expect, it } from 'vitest';+import { citationUrlMatchKey, normalizeCitationUrl } from './normalize.js';++describe('normalizeCitationUrl', () => {+ it('preserves the YouTube video id while discarding tracking parameters and fragments', () => {+ expect(+ normalizeCitationUrl(+ 'https://www.youtube.com/watch?utm_source=answer-engine&v=abc123&feature=shared#details',+ ),+ ).toBe('https://www.youtube.com/watch?v=abc123');+ expect(normalizeCitationUrl('https://m.youtube.com/watch?v=mobile&utm_medium=referral')).toBe(+ 'https://m.youtube.com/watch?v=mobile',+ );+ });++ it('keeps different YouTube videos in distinct buckets', () => {+ expect(normalizeCitationUrl('https://youtube.com/watch?v=abc')).not.toBe(+ normalizeCitationUrl('https://youtube.com/watch?v=def'),+ );+ });++ it('still strips query parameters from non-query-keyed URLs', () => {+ expect(normalizeCitationUrl('https://example.com/article/?utm_source=test#section')).toBe(+ 'https://example.com/article',+ );+ expect(normalizeCitationUrl('https://youtube.com/channel/example?view_as=subscriber')).toBe(+ 'https://youtube.com/channel/example',+ );+ });++ it('leaves path-keyed YouTube URLs intact', () => {+ expect(normalizeCitationUrl('https://youtu.be/abc123?si=tracking')).toBe(+ 'https://youtu.be/abc123',+ );+ expect(normalizeCitationUrl('https://youtube.com/shorts/abc123?feature=share')).toBe(+ 'https://youtube.com/shorts/abc123',+ );+ });++ it('returns unparseable input unchanged', () => {+ expect(normalizeCitationUrl('not a URL')).toBe('not a URL');+ });+});++describe('citationUrlMatchKey', () => {+ it('matches URLs loosely by normalized host and path', () => {+ expect(citationUrlMatchKey('https://www.Example.com/article/?utm_source=test#section')).toBe(+ 'example.com/article',+ );+ });++ it('returns null for unparseable input', () => {+ expect(citationUrlMatchKey('not a URL')).toBeNull();+ });+});diff --git a/web/src/lib/citations/normalize.ts b/web/src/lib/citations/normalize.tsnew file mode 100644index 000000000..bfb11b5b2--- /dev/null+++ b/web/src/lib/citations/normalize.ts@@ -0,0 +1,55 @@+/**+ * Identity-bearing query parameters for citation URLs.+ *+ * Hostnames are matched after removing a leading `www.`. Keep this table+ * deliberately small: query parameters are discarded unless a host/path pair+ * explicitly identifies them as part of the page identity.+ */+const IDENTITY_QUERY_PARAMS: Readonly<Record<string, Readonly<Record<string, readonly string[]>>>> =+ {+ 'youtube.com': { '/watch': ['v'] },+ 'm.youtube.com': { '/watch': ['v'] },+ };++/**+ * Canonical URL key for citation aggregation.+ *+ * Query parameters and fragments are normally discarded, and one trailing+ * slash is stripped. Known query-keyed pages retain only the parameters that+ * identify the page.+ */+export function normalizeCitationUrl(raw: string): string {+ try {+ const parsed = new URL(raw);+ const host = parsed.hostname.toLowerCase().replace(/^www\./, '');+ const identityParams = IDENTITY_QUERY_PARAMS[host]?.[parsed.pathname] ?? [];+ const preservedParams = identityParams.flatMap((param) =>+ parsed.searchParams.getAll(param).map((value) => [param, value] as const),+ );++ parsed.search = '';+ for (const [param, value] of preservedParams) {+ parsed.searchParams.append(param, value);+ }+ parsed.hash = '';++ return parsed.toString().replace(/\/$/, '');+ } catch {+ return raw;+ }+}++/**+ * Looser cross-source match key: host without `www.` plus path without+ * trailing slashes. This intentionally ignores protocol, query and fragment.+ */+export function citationUrlMatchKey(raw: string): string | null {+ try {+ const parsed = new URL(raw.trim());+ const host = parsed.hostname.toLowerCase().replace(/^www\./, '');+ const path = parsed.pathname.replace(/\/+$/, '');+ return `${host}${path}`;+ } catch {+ return null;+ }+}