Full Toolkit for Next-Level Domain Analysis
Align the for-sale lookup with RFC 10023 📜🏷️ (#118)
Felix Wotschofsky9a41cee272ae
diff --git a/app/lookup/[domain]/_components/domain-summary.test.tsx b/app/lookup/[domain]/_components/domain-summary.test.tsxindex d9d909e..fa6ac00 100644--- a/app/lookup/[domain]/_components/domain-summary.test.tsx+++ b/app/lookup/[domain]/_components/domain-summary.test.tsx@@ -183,6 +183,19 @@ describe('domain summary sale link', () => {expect(container.querySelector('a span')?.textContent).toBe('Advertised for sale',);+ expect(container.textContent).not.toContain('indicative');+ });++ it.each([+ { links: ['https://seller.example/buy'] },+ { links: ['mailto:[email protected]'] },+ { links: [] },+ ])('labels an advertised price as indicative: $links', async ({ links }) => {+ fixtures.listing.links = links;+ await render();+ expect(+ [...container.querySelectorAll('h3')].map((title) => title.textContent),+ ).toContain('For sale · indicative price');});it('prefers a website listing when an email address is also available', async () => {@@ -220,6 +233,19 @@ describe('domain summary sale link', () => {expect(container.textContent).toContain('For sale (example.com)');});+ it('omits the suffix when the listing is on the searched subdomain itself', async () => {+ fixtures.summary.mockReturnValue({+ data: {+ whois: { registered: true, registrar: 'Example registrar' },+ sale: { domain: 'dynamic.example.com', listing: fixtures.listing },+ },+ isLoading: false,+ });+ await render('dynamic.example.com');+ expect(container.textContent).toContain('For sale');+ expect(container.textContent).not.toContain('(dynamic.example.com)');+ });+it('shows a retryable error instead of loading forever, then recovers', async () => {const success = fixtures.summary();fixtures.summary.mockReturnValue({diff --git a/app/lookup/[domain]/_components/domain-summary.tsx b/app/lookup/[domain]/_components/domain-summary.tsxindex 96fa595..39e8fe5 100644--- a/app/lookup/[domain]/_components/domain-summary.tsx+++ b/app/lookup/[domain]/_components/domain-summary.tsx@@ -167,7 +167,9 @@ export const DomainSummary: FC<{ domain: string }> = ({ domain }) => {<DomainSummaryTile title="DNSSEC" value={whois.dnssec || 'Unavailable'} />{sale && listing && (<DomainSummaryTile- title={`For sale${sale.domain !== canonicalDnsName(domain) ? ` (${sale.domain})` : ''}`}+ // Advertised prices are not binding and are labelled as such, as+ // RFC 10023 (§4) asks of anyone displaying them.+ title={`For sale${sale.domain !== canonicalDnsName(domain) ? ` (${sale.domain})` : ''}${listing.prices.length ? ' · indicative price' : ''}`}value={<SaleListingLink href={saleUrl}>{listing.prices.lengthdiff --git a/lib/for-sale.test.ts b/lib/for-sale.test.tsindex 7ae0b75..48ecbb2 100644--- a/lib/for-sale.test.ts+++ b/lib/for-sale.test.ts@@ -45,7 +45,7 @@ describe('RFC 10023 records', () => {'v=FORSALE1; fval=EUR2500',]),).toEqual({- prices: ['EUR 2500', 'BTC 0.000010'],+ prices: ['BTC 0.000010', 'EUR 2500'],links: ['https://seller.example/buy','mailto:[email protected]',@@ -55,6 +55,68 @@ describe('RFC 10023 records', () => {});});+ it('orders details the same whatever order DNS returns the records in', () => {+ const records = [+ 'v=FORSALE1;fval=USD900',+ 'v=FORSALE1;fval=EUR1000',+ 'v=FORSALE1;fval=EUR250.5',+ 'v=FORSALE1;fval=BTC0.5',+ 'v=FORSALE1;fval=BTC0.25',+ `v=FORSALE1;fval=JPY1${'0'.repeat(30)}`,+ `v=FORSALE1;fval=JPY${'9'.repeat(30)}`,+ 'v=FORSALE1;fval=JPY9007199254740993',+ 'v=FORSALE1;fval=JPY9007199254740992',+ 'v=FORSALE1;furi=tel:+4930123456',+ 'v=FORSALE1;furi=mailto:[email protected]',+ 'v=FORSALE1;furi=http://seller.example/plain',+ 'v=FORSALE1;furi=https://seller.example/b',+ 'v=FORSALE1;furi=https://seller.example/a',+ 'v=FORSALE1;ftxt=Second',+ 'v=FORSALE1;ftxt=First',+ ];+ const sorted = {+ prices: [+ 'BTC 0.25',+ 'BTC 0.5',+ 'EUR 250.5',+ 'EUR 1000',+ 'JPY 9007199254740992',+ 'JPY 9007199254740993',+ `JPY ${'9'.repeat(30)}`,+ `JPY 1${'0'.repeat(30)}`,+ 'USD 900',+ ],+ links: [+ 'https://seller.example/a',+ 'https://seller.example/b',+ 'http://seller.example/plain',+ 'mailto:[email protected]',+ 'tel:+4930123456',+ ],+ texts: ['First', 'Second'],+ };+ expect(parseForSaleRecords(records)).toEqual(sorted);+ expect(parseForSaleRecords(records.toReversed())).toEqual(sorted);+ });++ it('lists a website on a plain host before an internationalized one', () => {+ expect(+ parseForSaleRecords([+ 'v=FORSALE1;furi=mailto:[email protected]',+ 'v=FORSALE1;furi=http://seller.example/plain',+ 'v=FORSALE1;furi=https://example.com.xn--9db.xn--9dbq2a/',+ 'v=FORSALE1;furi=https://xn--80ak6aa92e.com/',+ 'v=FORSALE1;furi=https://example.nl/for-sale.txt',+ ])?.links,+ ).toEqual([+ 'https://example.nl/for-sale.txt',+ 'https://example.com.xn--9db.xn--9dbq2a/',+ 'https://xn--80ak6aa92e.com/',+ 'http://seller.example/plain',+ 'mailto:[email protected]',+ ]);+ });+it('treats a semicolon in content as part of the value, not another tag', () => {expect(parseForSaleRecords(['v=FORSALE1;ftxt=Contact me;fval=EUR500']),@@ -74,6 +136,11 @@ describe('RFC 10023 records', () => {'https://seller.example/\n','https://seller.example/\u202etest','https://seller.example/\u061ctest',+ 'https://seller.example;http://other.example',+ 'https://*.seller.example/',+ 'https://192.0.2.1/',+ 'https://[2001:db8::1]/',+ 'http://localhost/',])('does not expose unsafe links: %s', (link) => {expect(parseForSaleRecords([`v=FORSALE1;furi=${link}`])?.links).toEqual([]);});@@ -117,10 +184,10 @@ describe('RFC 10023 records', () => {).toEqual([]);});- it('ignores oversized records before recognizing a sale signal', () => {- expect(- parseForSaleRecords([`v=FORSALE1;ftxt=${'€'.repeat(100)}`]),- ).toBeNull();+ it('keeps the sale signal of an oversized record but not its content', () => {+ expect(parseForSaleRecords([`v=FORSALE1;ftxt=${'€'.repeat(100)}`])).toEqual(+ { prices: [], links: [], texts: [] },+ );});it('enforces the 255-octet boundary using UTF-8 byte length', () => {@@ -128,7 +195,7 @@ describe('RFC 10023 records', () => {const record = `v=FORSALE1;ftxt=${text}`;expect(new TextEncoder().encode(record).length).toBe(255);expect(parseForSaleRecords([record])?.texts).toEqual([text]);- expect(parseForSaleRecords([record + 'x'])).toBeNull();+ expect(parseForSaleRecords([record + 'x'])?.texts).toEqual([]);});it('keeps valid records when another answer is oversized', () => {@@ -155,7 +222,18 @@ describe('for-sale lookup', () => {data,});const respond = (body: unknown, status = 200) =>- fetchMock.mockResolvedValue(new Response(JSON.stringify(body), { status }));+ fetchMock.mockImplementation(+ async () => new Response(JSON.stringify(body), { status }),+ );+ // Answer each queried name from its own record set.+ const respondByName = (records: Record<string, unknown>) =>+ fetchMock.mockImplementation(async (url: URL) => {+ const body = records[url.searchParams.get('name')!] ?? { Status: 3 };+ if (body instanceof Error) throw body;+ return new Response(JSON.stringify(body));+ });+ const queriedNames = () =>+ fetchMock.mock.calls.map(([url]) => url.searchParams.get('name'));beforeEach(() => {fetchMock.mockReset();@@ -163,12 +241,13 @@ describe('for-sale lookup', () => {});afterEach(() => vi.unstubAllGlobals());- it('looks up the registrable base domain with a deadline and no upstream cache', async () => {+ it('queries the searched name with a deadline and no upstream cache', async () => {respond({ Status: 0, Answer: [answer('"v=FORSALE1;fval=EUR2500"')] });- expect(await lookupForSale('*.WWW.Example.com.')).toEqual({+ expect(await lookupForSale('Example.com.')).toEqual({domain: 'example.com',listing: { prices: ['EUR 2500'], links: [], texts: [] },});+ expect(fetchMock).toHaveBeenCalledOnce();const [url, options] = fetchMock.mock.calls[0];expect(url.toString()).toBe('https://cloudflare-dns.com/dns-query?name=_for-sale.example.com&type=TXT',@@ -178,6 +257,78 @@ describe('for-sale lookup', () => {expect(options.signal).toBeInstanceOf(AbortSignal);});+ const eur = { Status: 0, Answer: [answer('"v=FORSALE1;fval=EUR3000"')] };+ const usd = { Status: 0, Answer: [answer('"v=FORSALE1;fval=USD1000000"')] };++ it('prefers a listing on the searched name over its registrable domain', async () => {+ respondByName({+ '_for-sale.dynamic.example.com': eur,+ '_for-sale.example.com': usd,+ });+ expect(await lookupForSale('*.Dynamic.Example.com.')).toEqual({+ domain: 'dynamic.example.com',+ listing: { prices: ['EUR 3000'], links: [], texts: [] },+ });+ expect(queriedNames().toSorted()).toEqual([+ '_for-sale.dynamic.example.com',+ '_for-sale.example.com',+ ]);+ });++ it.each(['shop.example.co.nl', 'shop.city.nl.eu.org', 'shop.town.ny.us'])(+ 'finds a listing below the registrable domain: %s',+ async (domain) => {+ respondByName({ [`_for-sale.${domain}`]: eur });+ expect(await lookupForSale(domain)).toEqual({+ domain,+ listing: { prices: ['EUR 3000'], links: [], texts: [] },+ });+ },+ );++ it('falls back to the registrable domain when the searched name has no listing', async () => {+ respondByName({ '_for-sale.example.com': usd });+ expect(await lookupForSale('www.example.com')).toEqual({+ domain: 'example.com',+ listing: { prices: ['USD 1000000'], links: [], texts: [] },+ });+ });++ it('keeps the listing on the searched name when the fallback lookup fails', async () => {+ respondByName({+ '_for-sale.dynamic.example.com': eur,+ '_for-sale.example.com': new DOMException('Timed out', 'TimeoutError'),+ });+ expect((await lookupForSale('dynamic.example.com')).domain).toBe(+ 'dynamic.example.com',+ );+ });++ it.each([{ Status: 2 }, new DOMException('Timed out', 'TimeoutError')])(+ 'does not fall back while the lookup of the searched name has failed: %j',+ async (failure) => {+ respondByName({+ '_for-sale.shop.example.com': failure,+ '_for-sale.example.com': usd,+ });+ await expect(lookupForSale('shop.example.com')).rejects.toThrow();+ },+ );++ it('rejects when the fallback lookup fails and the searched name has no listing', async () => {+ respondByName({ '_for-sale.example.com': { Status: 2 } });+ await expect(lookupForSale('www.example.com')).rejects.toThrow();+ });++ it('queries only the registrable domain when the searched name cannot take the label', async () => {+ const domain = `${'a'.repeat(63)}.${'b'.repeat(63)}.${'c'.repeat(63)}.${'d'.repeat(40)}.example.com`;+ expect(domain.length).toBeLessThanOrEqual(253);+ expect(`_for-sale.${domain}`.length).toBeGreaterThan(253);+ respondByName({ '_for-sale.example.com': usd });+ expect((await lookupForSale(domain)).domain).toBe('example.com');+ expect(queriedNames()).toEqual(['_for-sale.example.com']);+ });+it('decodes escaped quotes, backslashes and UTF-8 octets', async () => {respond({Status: 0,@@ -245,13 +396,35 @@ describe('for-sale lookup', () => {'"v=FORSALE1;ftxt=\\999"','"v=FORSALE1;" garbage','"v=FORSALE1;',- '"v=FORSALE1;" "fval=USD10"',- '"v=FOR" "SALE1;"',+ '"v=FORSALE1;" x "fval=USD10"',+ '"v=FORSALE1;""fval=USD10"',])('ignores malformed presentation data: %s', async (data) => {respond({ Status: 0, Answer: [answer(data)] });expect((await lookupForSale('example.com')).listing).toBeNull();});+ it.each([+ ['"v=FORSALE1;" "fval=USD10"', { prices: ['USD 10'], texts: [] }],+ ['"v=FOR" "SALE1;"', { prices: [], texts: [] }],+ ['"v=FORSALE1;ftxt=foo" " bar"', { prices: [], texts: ['foo bar'] }],+ // A UTF-8 sequence may be split across character-strings.+ [+ String.raw`"v=FORSALE1;ftxt=\240" "\159\142\133"`,+ { prices: [], texts: ['🎅'] },+ ],+ // An escaped digit must not merge with digits of the next string.+ [String.raw`"v=FORSALE1;ftxt=\1" "23"`, { prices: [], texts: ['123'] }],+ ])(+ 'joins multiple character-strings before reading the record: %s',+ async (data, expected) => {+ respond({ Status: 0, Answer: [answer(data)] });+ expect((await lookupForSale('example.com')).listing).toEqual({+ links: [],+ ...expected,+ });+ },+ );+it.each(['fcod', 'ftxt', 'furi', 'fval'])('preserves the sale marker when %s contains invalid UTF-8',async (tag) => {@@ -281,11 +454,16 @@ describe('for-sale lookup', () => {`"v=FORSALE1;ftxt=${'x'.repeat(240)}"`,`"v=FORSALE1;ftxt=${String.raw`\195\188`.repeat(120)}"`,`"v=FORSALE1;fcod=${String.raw`\255`.repeat(240)}"`,+ `"v=FORSALE1;ftxt=${'x'.repeat(200)}" "${'x'.repeat(200)}"`,])(- 'ignores overlong quoted records, including invalid optional UTF-8',+ 'keeps only the sale signal of overlong records, including invalid optional UTF-8',async (data) => {respond({ Status: 0, Answer: [answer(data)] });- expect((await lookupForSale('example.com')).listing).toBeNull();+ expect((await lookupForSale('example.com')).listing).toEqual({+ prices: [],+ links: [],+ texts: [],+ });},);@@ -314,7 +492,11 @@ describe('for-sale lookup', () => {),),});- expect((await lookupForSale('example.com')).listing).toBeNull();+ expect((await lookupForSale('example.com')).listing).toEqual({+ prices: [],+ links: [],+ texts: [],+ });});it.each(diff --git a/lib/for-sale.ts b/lib/for-sale.tsindex 59b6876..e46063c 100644--- a/lib/for-sale.ts+++ b/lib/for-sale.ts@@ -1,3 +1,4 @@+import { canonicalDnsName } from './resolvers/base';import { CloudflareDoHResolver } from './resolvers/cloudflare';import { getBaseDomain, isValidDomain } from './utils';@@ -5,6 +6,12 @@ const VERSION = 'v=FORSALE1;';const MAX_ANSWERS = 256;const MAX_ANSWER_BYTES = 4096;const UNSAFE_CHARACTERS = /[\u0000-\u001f\u007f-\u009f\p{Bidi_Control}]/gu;+// The schemes RFC 10023 recommends (§2.2.3), most preferred first (§4).+const LINK_SCHEMES = ['https:', 'http:', 'mailto:', 'tel:'];+// A dotted DNS name as the URL parser serializes it: no IP address, no+// single-label or wildcard host.+const HOSTNAME =+ /^(?:[a-z0-9_](?:[a-z0-9_-]{0,61}[a-z0-9_])?\.)+[a-z](?:[a-z0-9-]{0,61}[a-z0-9])?\.?$/;export type ForSaleListing = {prices: string[];@@ -22,26 +29,60 @@ const safeLink = (value: string): string | null => {return null;try {const url = new URL(value);- if (!['http:', 'https:', 'mailto:', 'tel:'].includes(url.protocol))- return null;+ if (!LINK_SCHEMES.includes(url.protocol)) return null;if (url.username || url.password || (!url.pathname && !url.hostname))return null;+ // "https://a.example;http://b.example" parses as one URL with a bogus host.+ if (url.protocol.startsWith('http') && !HOSTNAME.test(url.hostname))+ return null;return url.href;} catch {return null;}};+const compare = (a: string | number, b: string | number) =>+ a < b ? -1 : a > b ? 1 : 0;++// Order decimal strings by value without converting them to floats.+const compareAmounts = (a: string, b: string) => {+ const [integerA, fractionA = ''] = a.replace(/^0+(?=\d)/, '').split('.');+ const [integerB, fractionB = ''] = b.replace(/^0+(?=\d)/, '').split('.');+ return (+ compare(integerA.length, integerB.length) ||+ compare(integerA, integerB) ||+ compare(fractionA.replace(/0+$/, ''), fractionB.replace(/0+$/, ''))+ );+};++// DNS returns an RRset in no particular order, so details are sorted to show+// the same listing on every lookup: prices by currency, then amount.+const comparePrices = (a: string, b: string) => {+ const [currencyA, amountA] = a.split(' ');+ const [currencyB, amountB] = b.split(' ');+ return (+ compare(currencyA, currencyB) ||+ compareAmounts(amountA, amountB) ||+ compare(a, b)+ );+};++// Within a scheme, websites on a plain host come before internationalized+// ones: lookalike names are a risk there (§4), and their punycode form is+// harder to check before opening the link.+const compareLinks = (a: string, b: string) => {+ const preference = (link: string) =>+ 2 * LINK_SCHEMES.findIndex((scheme) => link.startsWith(scheme)) ++ Number(/^https?:\/\/(?:[^/?#]*\.)?xn--/.test(link));+ return compare(preference(a), preference(b)) || compare(a, b);+};+// Parse decoded TXT RDATA, not DNS presentation strings. A valid version alone// is a sale signal, even when the optional content is unknown or invalid (§2.1).export const parseForSaleRecords = (records: string[],): ForSaleListing | null => {- const saleRecords = records.filter(- (record) =>- record.startsWith(VERSION) &&- new TextEncoder().encode(record).length <= 255,- );+ const saleRecords = records.filter((record) => record.startsWith(VERSION));if (!saleRecords.length) return null;const prices = new Set<string>();@@ -49,6 +90,10 @@ export const parseForSaleRecords = (const texts = new Set<string>();for (const record of saleRecords) {+ // A valid record fits one character-string of 255 octets (§2.4), so a+ // longer one, counted over all of its joined strings, loses its content.+ if (new TextEncoder().encode(record).length > 255) continue;+const content = record.slice(VERSION.length).trimStart();const separator = content.indexOf('=');if (separator === -1) continue;@@ -67,38 +112,42 @@ export const parseForSaleRecords = (}// fcod is a cooperating party's opaque code, not a URL we can decode.}- return { prices: [...prices], links: [...links], texts: [...texts] };+ return {+ prices: [...prices].sort(comparePrices),+ links: [...links].sort(compareLinks),+ texts: [...texts].sort(compare),+ };};// Accept plain TXT content or quoted DNS presentation data from DoH resolvers.-// Sale records must contain a single character-string (§2.4).-// Decode decimal octet escapes before interpreting their UTF-8 content.+// Decode decimal octet escapes and join multiple character-strings before+// interpreting the octets as UTF-8 (§3.2).const decodeTxt = (data: string): string | null => {if (!data.startsWith('"')) return data;- const match = /^"((?:[^"\\]|\\[\s\S])*)"$/u.exec(data);- if (!match) return null;- const content = match[1];+ const strings = [...data.matchAll(/"((?:[^"\\]|\\[\s\S])*)"(?: |$)/guy)];+ if (strings.map(([string]) => string).join('') !== data) return null;+const bytes: number[] = [];- for (let i = 0; i < content.length; ) {- if (content[i] === '\\') {- const decimal = content.slice(i + 1, i + 4);- if (/^[0-9]{3}$/.test(decimal)) {- const octet = Number(decimal);- if (octet > 255) return null;- bytes.push(octet);- i += 4;- continue;+ for (const [, content] of strings) {+ for (let i = 0; i < content.length; ) {+ if (content[i] === '\\') {+ const decimal = content.slice(i + 1, i + 4);+ if (/^[0-9]{3}$/.test(decimal)) {+ const octet = Number(decimal);+ if (octet > 255) return null;+ bytes.push(octet);+ i += 4;+ continue;+ }+ i++;}- i++;+ const character = String.fromCodePoint(content.codePointAt(i)!);+ bytes.push(...new TextEncoder().encode(character));+ i += character.length;}- const character = String.fromCodePoint(content.codePointAt(i)!);- bytes.push(...new TextEncoder().encode(character));- i += character.length;}- if (bytes.length > 255) return null;-try {return new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }).decode(new Uint8Array(bytes),@@ -113,20 +162,12 @@ const decodeTxt = (data: string): string | null => {}};-export const lookupForSale = async (- domain: string,-): Promise<ForSaleSummary> => {- if (!isValidDomain(domain)) throw new Error('Invalid domain');-- const baseDomain = getBaseDomain(domain).toLowerCase();- if (baseDomain === 'arpa' || baseDomain.endsWith('.arpa'))- return { domain: baseDomain, listing: null };-- const resolver = new CloudflareDoHResolver({- signal: AbortSignal.timeout(2500),- });+const lookupListing = async (+ resolver: CloudflareDoHResolver,+ name: string,+): Promise<ForSaleListing | null> => {const { answers, rcode } = await resolver.resolveAnswers(- `_for-sale.${baseDomain}`,+ `_for-sale.${name}`,'TXT',);if (@@ -143,5 +184,38 @@ export const lookupForSale = async (.filter((answer) => answer.type === 16).map((answer) => decodeTxt(answer.data)).filter((record): record is string => record !== null);- return { domain: baseDomain, listing: parseForSaleRecords(records) };+ return parseForSaleRecords(records);+};++export const lookupForSale = async (+ domain: string,+): Promise<ForSaleSummary> => {+ if (!isValidDomain(domain)) throw new Error('Invalid domain');++ const baseDomain = getBaseDomain(domain).toLowerCase();+ if (baseDomain === 'arpa' || baseDomain.endsWith('.arpa'))+ return { domain: baseDomain, listing: null };++ // The leaf may sit at any level of the DNS (§2.6), so a record on the+ // searched name comes first. The registrable domain is the only fallback:+ // www.example.com still shows that example.com is for sale, but no name+ // in between is queried.+ const name = canonicalDnsName(domain).replace(/^\*\./, '');+ const names = [...new Set([name, baseDomain])].filter(+ (candidate) => `_for-sale.${candidate}`.length <= 253,+ );++ const resolver = new CloudflareDoHResolver({+ signal: AbortSignal.timeout(2500),+ });+ const results = await Promise.allSettled(+ names.map((candidate) => lookupListing(resolver, candidate)),+ );+ for (const [index, result] of results.entries()) {+ // The fallback only counts once the searched name is known to have no+ // listing, so a failed lookup of that name never shows the fallback's.+ if (result.status === 'rejected') throw result.reason;+ if (result.value) return { domain: names[index], listing: result.value };+ }+ return { domain: baseDomain, listing: null };};