Skip to content

web search done right

13.1 KB
Raw Blame
/* eslint-disable @typescript-eslint/no-unused-vars */

type Method = 'DELETE' | 'GET' | 'PUT' | 'POST' | 'HEAD' | 'TRACE' | 'PATCH';

let GLOBAL_API_BASE = '';
export const getApiBase = (options?: ApiOptions) => options?.apiBase ?? GLOBAL_API_BASE;
export const setGlobalApiBase = (apiBase: string) => (GLOBAL_API_BASE = apiBase);

export type ApiOptions = {
  fetch?: typeof fetch;
  apiBase?: string;
  headers?: Record<string, string>;
};

export const requestPlain = (
  method: Method,
  url: string,
  body?: unknown,
  options?: ApiOptions,
): {
  data: Promise<string>;
  cancel: (reason?: string) => void;
} => {
  let inFlight = true;
  const controller = new AbortController();
  const data = (options?.fetch ?? fetch)(`${getApiBase(options)}${url}`, {
    method: method.toUpperCase(),
    body: typeof body != 'undefined' ? JSON.stringify(body) : void 0,
    signal: controller.signal,
    headers: {
      ...options?.headers,
      ...(typeof body != 'undefined' ? { 'Content-Type': 'application/json' } : {}),
    },
  }).then(async (res) => {
    inFlight = false;
    if (res.ok) {
      const text = await res.text();
      try {
        return text;
      } catch (_) {
        throw text;
      }
    } else {
      throw res.text();
    }
  });

  return {
    data,
    cancel: (reason) => {
      if (inFlight) controller.abort(reason);
    },
  };
};

export const requestJson = <T>(
  method: Method,
  url: string,
  body?: unknown,
  options: ApiOptions = {},
): {
  data: Promise<T>;
  cancel: (reason?: string) => void;
} => {
  const { data, cancel } = requestPlain(method, url, body, options);
  return { data: data.then((text) => JSON.parse(text) as T), cancel };
};

export type SSEStream<T> = (
  event:
    | { type: 'message'; data: T }
    | {
        type: 'error';
        event: Event;
      },
) => void;

const sse = <T>(
  _method: Method,
  url: string,
  options?: ApiOptions,
): {
  cancel: () => void;
  listen: (stream: SSEStream<T>) => void;
} => {
  const source = new EventSource(`${getApiBase(options)}${url}`);

  let stream: SSEStream<T> | null = null;

  source.onmessage = (event) => {
    const data = event.data;
    stream?.({ type: 'message', data });
  };
  source.onerror = (event) => {
    stream?.({ type: 'error', event });
  };
  return {
    cancel: () => source.close(),
    listen: (newStream) => (stream = newStream),
  };
};

export const api = {
  autosuggest: (
    params: {
      q: string;
    },
    options?: ApiOptions,
  ) =>
    requestJson<Suggestion[]>(
      'POST',
      `/beta/api/autosuggest?${new URLSearchParams(params)}`,
      options,
    ),
  exploreExport: (body: ExploreExportOpticParams, options?: ApiOptions) =>
    requestPlain('POST', `/beta/api/explore/export`, body, options),
  hostsExport: (body: HostsExportOpticParams, options?: ApiOptions) =>
    requestPlain('POST', `/beta/api/hosts/export`, body, options),
  search: (body: ApiSearchQuery, options?: ApiOptions) =>
    requestJson<ApiSearchResult>('POST', `/beta/api/search`, body, options),
  searchSidebar: (body: SidebarQuery, options?: ApiOptions) =>
    requestJson<null | DisplayedSidebar>('POST', `/beta/api/search/sidebar`, body, options),
  searchSpellcheck: (body: SpellcheckQuery, options?: ApiOptions) =>
    requestJson<null | HighlightedSpellCorrection>(
      'POST',
      `/beta/api/search/spellcheck`,
      body,
      options,
    ),
  searchWidget: (body: WidgetQuery, options?: ApiOptions) =>
    requestJson<null | Widget>('POST', `/beta/api/search/widget`, body, options),
  webgraphHostIngoing: (
    query: {
      host: string;
    },
    options?: ApiOptions,
  ) =>
    requestJson<PrettyEdge[]>(
      'POST',
      `/beta/api/webgraph/host/ingoing?${new URLSearchParams(query)}`,
      options,
    ),
  webgraphHostKnows: (
    query: {
      host: string;
    },
    options?: ApiOptions,
  ) =>
    requestJson<KnowsHost>(
      'POST',
      `/beta/api/webgraph/host/knows?${new URLSearchParams(query)}`,
      options,
    ),
  webgraphHostOutgoing: (
    query: {
      host: string;
    },
    options?: ApiOptions,
  ) =>
    requestJson<PrettyEdge[]>(
      'POST',
      `/beta/api/webgraph/host/outgoing?${new URLSearchParams(query)}`,
      options,
    ),
  webgraphHostSimilar: (body: SimilarHostsQuery, options?: ApiOptions) =>
    requestJson<ScoredHost[]>('POST', `/beta/api/webgraph/host/similar`, body, options),
  webgraphPageIngoing: (
    query: {
      page: string;
    },
    options?: ApiOptions,
  ) =>
    requestJson<PrettyEdge[]>(
      'POST',
      `/beta/api/webgraph/page/ingoing?${new URLSearchParams(query)}`,
      options,
    ),
  webgraphPageOutgoing: (
    query: {
      page: string;
    },
    options?: ApiOptions,
  ) =>
    requestJson<PrettyEdge[]>(
      'POST',
      `/beta/api/webgraph/page/outgoing?${new URLSearchParams(query)}`,
      options,
    ),
};

export type ApiSearchQuery = {
  countResultsExact?: boolean;
  flattenResponse?: boolean;
  hostRankings?: null | HostRankings;
  numResults?: number;
  optic?: string;
  page?: number;
  query: string;
  returnRankingSignals?: boolean;
  returnStructuredData?: boolean;
  safeSearch?: boolean;
  selectedRegion?: null | Region;
  signalCoefficients?: {};
};
export type ApiSearchResult =
  | (WebsitesResult & {
      _type: 'websites';
    })
  | (BangHit & {
      _type: 'bang';
    });
export type Bang = {
  c?: string;
  d?: string;
  r?: number;
  s?: string;
  sc?: string;
  t: string;
  u: string;
};
export type BangHit = {
  bang: Bang;
  redirectTo: UrlWrapper;
};
export type Calculation = {
  input: string;
  result: string;
};
export type CodeOrText =
  | {
      _type: 'code';
      value: string;
    }
  | {
      _type: 'text';
      value: string;
    };
export type Count =
  | {
      _type: 'exact';
      value: number;
    }
  | {
      _type: 'approximate';
      value: number;
    };
export type Definition = string;
export type DisplayedAnswer = {
  answer: string;
  prettyUrl: string;
  snippet: string;
  title: string;
  url: string;
};
export type DisplayedEntity = {
  imageId?: string;
  info: [
    string,
    {
      fragments: EntitySnippetFragment[];
    },
  ][];
  matchScore: number;
  relatedEntities: DisplayedEntity[];
  smallAbstract: EntitySnippet;
  title: string;
};
export type DisplayedSidebar =
  | {
      _type: 'entity';
      value: DisplayedEntity;
    }
  | {
      _type: 'stackOverflow';
      value: {
        answer: StackOverflowAnswer;
        title: string;
      };
    };
export type DisplayedWebpage = {
  domain: string;
  likelyHasAds: boolean;
  likelyHasPaywall: boolean;
  prettyUrl: string;
  rankingSignals?: {};
  richSnippet?: null | RichSnippet;
  site: string;
  snippet: Snippet;
  structuredData?: StructuredData[];
  title: string;
  url: string;
};
export type EntitySnippet = {
  fragments: EntitySnippetFragment[];
};
export type EntitySnippetFragment =
  | {
      kind: 'normal';
      text: string;
    }
  | {
      href: string;
      kind: 'link';
      text: string;
    };
export type Example = string;
export type ExploreExportOpticParams = {
  chosenHosts: string[];
  similarHosts: string[];
};
export type HighlightedFragment = {
  kind: HighlightedKind;
  text: string;
};
export type HighlightedKind = 'normal' | 'highlighted';
export const HIGHLIGHTED_KINDS = ['normal', 'highlighted'] satisfies HighlightedKind[];
export type HighlightedSpellCorrection = {
  highlighted: HighlightedFragment[];
  raw: string;
};
export type HostRankings = {
  blocked: string[];
  disliked: string[];
  liked: string[];
};
export type HostsExportOpticParams = {
  hostRankings: HostRankings;
};
export type KnowsHost =
  | {
      _type: 'known';
      host: string;
    }
  | {
      _type: 'unknown';
    };
export type Lemma = string;
export type Node = {
  name: string;
};
export type OneOrManyProperty = Property | Property[];
export type OneOrManyString = string | string[];
export type PartOfSpeech = 'noun' | 'verb' | 'adjective' | 'adjectiveSatellite' | 'adverb';
export const PART_OF_SPEECHES = [
  'noun',
  'verb',
  'adjective',
  'adjectiveSatellite',
  'adverb',
] satisfies PartOfSpeech[];
export type PartOfSpeechMeaning = {
  meanings: WordMeaning[];
  pos: PartOfSpeech;
};
export type PrettyEdge = {
  from: string;
  label: string;
  rel_flags: PrettyRelFlag[];
  to: string;
};
export type PrettyRelFlag =
  | 'alternate'
  | 'author'
  | 'canonical'
  | 'help'
  | 'icon'
  | 'license'
  | 'me'
  | 'next'
  | 'no_follow'
  | 'prev'
  | 'privacy_policy'
  | 'search'
  | 'stylesheet'
  | 'tag'
  | 'terms_of_service'
  | 'sponsored'
  | 'is_in_footer'
  | 'is_in_navigation'
  | 'link_tag'
  | 'script_tag'
  | 'meta_tag'
  | 'same_icann_domain';
export const PRETTY_REL_FLAGS = [
  'alternate',
  'author',
  'canonical',
  'help',
  'icon',
  'license',
  'me',
  'next',
  'no_follow',
  'prev',
  'privacy_policy',
  'search',
  'stylesheet',
  'tag',
  'terms_of_service',
  'sponsored',
  'is_in_footer',
  'is_in_navigation',
  'link_tag',
  'script_tag',
  'meta_tag',
  'same_icann_domain',
] satisfies PrettyRelFlag[];
export type Property = string | StructuredData;
export type Region = 'All' | 'Denmark' | 'France' | 'Germany' | 'Spain' | 'US';
export const REGIONS = ['All', 'Denmark', 'France', 'Germany', 'Spain', 'US'] satisfies Region[];
export type ReturnBody =
  | {
      _type: 'all';
    }
  | {
      _type: 'truncated';
      value: number;
    };
export type RichSnippet = {
  _type: 'stackOverflowQA';
  answers: StackOverflowAnswer[];
  question: StackOverflowQuestion;
};
export type ScoredHost = {
  description?: string;
  host: string;
  score: number;
};
export type SidebarQuery = {
  query: string;
};
export type SignalEnumDiscriminants =
  | 'bm25_f'
  | 'bm25_title'
  | 'title_coverage'
  | 'bm25_title_bigrams'
  | 'bm25_title_trigrams'
  | 'bm25_clean_body'
  | 'clean_body_coverage'
  | 'bm25_clean_body_bigrams'
  | 'bm25_clean_body_trigrams'
  | 'bm25_stemmed_title'
  | 'bm25_stemmed_clean_body'
  | 'bm25_all_body'
  | 'bm25_keywords'
  | 'bm25_backlink_text'
  | 'idf_sum_url'
  | 'idf_sum_site'
  | 'idf_sum_domain'
  | 'idf_sum_site_no_tokenizer'
  | 'idf_sum_domain_no_tokenizer'
  | 'idf_sum_domain_name_no_tokenizer'
  | 'idf_sum_domain_if_homepage'
  | 'idf_sum_domain_name_if_homepage_no_tokenizer'
  | 'idf_sum_domain_if_homepage_no_tokenizer'
  | 'idf_sum_title_if_homepage'
  | 'cross_encoder_snippet'
  | 'cross_encoder_title'
  | 'host_centrality'
  | 'host_centrality_rank'
  | 'page_centrality'
  | 'page_centrality_rank'
  | 'is_homepage'
  | 'fetch_time_ms'
  | 'update_timestamp'
  | 'tracker_score'
  | 'region'
  | 'query_centrality'
  | 'inbound_similarity'
  | 'lambda_mart'
  | 'url_digits'
  | 'url_slashes'
  | 'link_density'
  | 'title_embedding_similarity'
  | 'keyword_embedding_similarity'
  | 'has_ads'
  | 'min_title_slop'
  | 'min_clean_body_slop';
export const SIGNAL_ENUM_DISCRIMINANTS = [
  'bm25_f',
  'bm25_title',
  'title_coverage',
  'bm25_title_bigrams',
  'bm25_title_trigrams',
  'bm25_clean_body',
  'clean_body_coverage',
  'bm25_clean_body_bigrams',
  'bm25_clean_body_trigrams',
  'bm25_stemmed_title',
  'bm25_stemmed_clean_body',
  'bm25_all_body',
  'bm25_keywords',
  'bm25_backlink_text',
  'idf_sum_url',
  'idf_sum_site',
  'idf_sum_domain',
  'idf_sum_site_no_tokenizer',
  'idf_sum_domain_no_tokenizer',
  'idf_sum_domain_name_no_tokenizer',
  'idf_sum_domain_if_homepage',
  'idf_sum_domain_name_if_homepage_no_tokenizer',
  'idf_sum_domain_if_homepage_no_tokenizer',
  'idf_sum_title_if_homepage',
  'cross_encoder_snippet',
  'cross_encoder_title',
  'host_centrality',
  'host_centrality_rank',
  'page_centrality',
  'page_centrality_rank',
  'is_homepage',
  'fetch_time_ms',
  'update_timestamp',
  'tracker_score',
  'region',
  'query_centrality',
  'inbound_similarity',
  'lambda_mart',
  'url_digits',
  'url_slashes',
  'link_density',
  'title_embedding_similarity',
  'keyword_embedding_similarity',
  'has_ads',
  'min_title_slop',
  'min_clean_body_slop',
] satisfies SignalEnumDiscriminants[];
export type SignalScore = {
  coefficient: number;
  value: number;
};
export type SimilarHostsQuery = {
  filters?: string[];
  hosts: string[];
  topN: number;
};
export type Snippet = {
  date?: string;
  text: TextSnippet;
};
export type SpellcheckQuery = {
  query: string;
};
export type StackOverflowAnswer = {
  accepted: boolean;
  body: CodeOrText[];
  date: string;
  upvotes: number;
  url: string;
};
export type StackOverflowQuestion = {
  body: CodeOrText[];
};
export type StructuredData = {
  _type?: null | OneOrManyString;
};
export type Suggestion = {
  highlighted: HighlightedFragment[];
  raw: string;
};
export type TextSnippet = {
  fragments: HighlightedFragment[];
};
export type ThesaurusWidget = {
  meanings: PartOfSpeechMeaning[];
  term: Lemma;
};
export type UrlWrapper = string;
export type WebsitesResult = {
  hasMoreResults: boolean;
  numHits: Count;
  searchDurationMs: number;
  webpages: DisplayedWebpage[];
};
export type Widget =
  | {
      _type: 'calculator';
      value: Calculation;
    }
  | {
      _type: 'thesaurus';
      value: ThesaurusWidget;
    };
export type WidgetQuery = {
  query: string;
};
export type WordMeaning = {
  definition: Definition;
  examples: Example[];
  similar: Lemma[];
};