Skip to content

web search done right

host graph is now based on root domain

Mikkel Denker0b1ee0ec5292
diff --git a/core/src/crawler/planner.rs b/core/src/crawler/planner.rs
index a835da74..16daaa2a 100644
--- a/core/src/crawler/planner.rs
+++ b/core/src/crawler/planner.rs
@@ -152,7 +152,8 @@ pub fn make_crawl_plan<P: AsRef<Path>>(
let host_budget = ((config.crawl_budget as f64 * host_centrality)
/ total_host_centrality)
- .max(1.0) as u64;
+ .max(0.0)
+ .round() as u64;
tracing::debug!("host_budget: {host_budget}");
let schedule_budget = (host_budget as f64 * (1.0 - config.wander_fraction))
.max(0.0)
@@ -160,7 +161,6 @@ pub fn make_crawl_plan<P: AsRef<Path>>(
tracing::debug!("schedule_budget: {schedule_budget}");
let wander_budget = (host_budget as f64 * config.wander_fraction)
.max(0.0)
- .min(schedule_budget as f64)
.round() as u64;
tracing::debug!("wander_budget: {wander_budget}");
diff --git a/core/src/ranking/bitvec_similarity.rs b/core/src/ranking/bitvec_similarity.rs
index f859db60..f50effe7 100644
--- a/core/src/ranking/bitvec_similarity.rs
+++ b/core/src/ranking/bitvec_similarity.rs
@@ -20,7 +20,6 @@ use serde::{Deserialize, Serialize};
use crate::hyperloglog::HyperLogLog;
const THRESHOLD_SIM_ESTIMATE: f64 = 0.1;
-
const HYPERLOGLOG_REGISTERS: usize = 128;
#[derive(Serialize, Deserialize, Debug, Clone)]
diff --git a/core/src/search_prettifier/mod.rs b/core/src/search_prettifier/mod.rs
index d238da15..f1dcc7f8 100644
--- a/core/src/search_prettifier/mod.rs
+++ b/core/src/search_prettifier/mod.rs
@@ -169,7 +169,6 @@ pub struct DisplayedWebpage {
pub url: String,
pub site: String,
pub domain: String,
- pub host: String,
pub pretty_url: String,
pub snippet: Snippet,
pub ranking_signals: Option<HashMap<Signal, SignalScore>>,
@@ -191,7 +190,6 @@ impl From<RetrievedWebpage> for DisplayedWebpage {
let url = Url::parse(&webpage.url).unwrap();
let domain = url.root_domain().unwrap_or_default().to_string();
- let host = url.host_str().unwrap_or_default().to_string();
let pretty_url = prettify_url(&url);
Self {
@@ -200,7 +198,6 @@ impl From<RetrievedWebpage> for DisplayedWebpage {
url: webpage.url,
pretty_url,
domain,
- host,
snippet,
ranking_signals: None,
}
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index 5d17589a..428f4e29 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -28,6 +28,7 @@ use utoipa::ToSchema;
use crate::executor::Executor;
use crate::intmap;
+use crate::webpage::url_ext::UrlExt;
pub mod centrality;
mod store;
@@ -187,7 +188,7 @@ impl Node {
match url {
Ok(url) => {
- let host = url.host_str().unwrap_or_default().to_string();
+ let host = url.root_domain().unwrap_or_default().to_string();
Node { name: host }
}
Err(_) => Node {
diff --git a/frontend/src/lib/api/index.ts b/frontend/src/lib/api/index.ts
index 6f934291..31b1d35d 100644
--- a/frontend/src/lib/api/index.ts
+++ b/frontend/src/lib/api/index.ts
@@ -258,7 +258,6 @@ export type DisplayedSidebar =
};
export type DisplayedWebpage = {
domain: string;
- host: string;
prettyUrl: string;
rankingSignals?: {};
site: string;
diff --git a/frontend/src/lib/components/Code.svelte b/frontend/src/lib/components/Code.svelte
index b6da7948..96e93d01 100644
--- a/frontend/src/lib/components/Code.svelte
+++ b/frontend/src/lib/components/Code.svelte
@@ -1,13 +1,15 @@
<script lang="ts">
- import {HighlightAuto} from "svelte-highlight";
+ import { HighlightAuto } from 'svelte-highlight';
export let code: string;
export let transparentBackground = false;
</script>
-<div class="code rounded-lg {transparentBackground ? '' : 'bg-base-200'} text-sm text-neutral-focus">
+<div
+ class="code rounded-lg {transparentBackground ? '' : 'bg-base-200'} text-sm text-neutral-focus"
+>
<div class="overflow-auto px-3 py-2">
- <HighlightAuto code={code}/>
+ <HighlightAuto {code} />
</div>
</div>
diff --git a/frontend/src/lib/improvements.ts b/frontend/src/lib/improvements.ts
index 8f8456ea..68db224c 100644
--- a/frontend/src/lib/improvements.ts
+++ b/frontend/src/lib/improvements.ts
@@ -11,7 +11,14 @@ export const updateQueryId = async ({
}: {
query: string;
webpages: DisplayedWebpage[];
-}) => queryIdStore.set(await queryId({ query, urls: webpages.map((wp) => wp.url) }).data);
+}) => {
+ let allowStats: boolean | undefined;
+ allowStatsStore.subscribe((allow) => (allowStats = allow));
+
+ if (!allowStats) return;
+
+ queryIdStore.set(await queryId({ query, urls: webpages.map((wp) => wp.url) }).data);
+};
export const improvements: Action<HTMLAnchorElement, number> = (node, webpageIndex) => {
let queryId: string | undefined;
diff --git a/frontend/src/lib/optics.ts b/frontend/src/lib/optics.ts
index 0614fcf2..686ed48b 100644
--- a/frontend/src/lib/optics.ts
+++ b/frontend/src/lib/optics.ts
@@ -58,8 +58,7 @@ export const DEFAULT_OPTICS = [
},
{
url: 'https://raw.githubusercontent.com/StractOrg/sample-optics/main/tilde.optic',
- description:
- 'Only search in urls that contain a tilde (~).',
+ description: 'Only search in urls that contain a tilde (~).',
name: 'Tilde',
shown: false,
},
diff --git a/frontend/src/lib/stores.ts b/frontend/src/lib/stores.ts
index 37db7651..b718ecae 100644
--- a/frontend/src/lib/stores.ts
+++ b/frontend/src/lib/stores.ts
@@ -25,14 +25,24 @@ const writableLocalStorage = <T>(
? parseJSONWithFallback(storedValue, defaultValue, `Failed to parse value stored in '${key}'`)
: defaultValue,
);
+
store.subscribe(($value) => {
- if (storage)
+ if (browser && storage) {
if (typeof $value == 'undefined') {
storage.removeItem(key);
} else {
storage.setItem(key, JSON.stringify($value));
}
+ }
});
+
+ const { set } = store;
+ store.set = (value: T) => {
+ if (storage) {
+ set(value);
+ }
+ };
+
return store;
};
@@ -55,16 +65,12 @@ const SITE_RANKINGS_KEY = 'siteRankings';
export const siteRankingsStore = writableLocalStorage<SiteRakings>(SITE_RANKINGS_KEY, {});
const SEARCH_QUERY_KEY = 'searchQuery';
-export const searchQueryStore = writableLocalStorage<string | undefined>(
- SEARCH_QUERY_KEY,
- void 0,
- browser && sessionStorage,
-);
+export const searchQueryStore = writableLocalStorage<string | undefined>(SEARCH_QUERY_KEY, void 0);
const THEME_KEY = 'theme';
export const themeStore = writableLocalStorage<string | void>(THEME_KEY, void 0);
if (browser)
- themeStore.subscribe(($theme) => {
+ themeStore?.subscribe(($theme) => {
const c = document.documentElement.className.replace(/theme-[^ ]+/, ``);
const theme = $theme?.toLowerCase() || '';
document.documentElement.className = `${c} ${theme}`.trim();
diff --git a/frontend/src/routes/search/+page.server.ts b/frontend/src/routes/search/+page.server.ts
index 5b3cd94c..68f8c2d5 100644
--- a/frontend/src/routes/search/+page.server.ts
+++ b/frontend/src/routes/search/+page.server.ts
@@ -15,12 +15,12 @@ export const load = async ({ locals, fetch, url, getClientAddress }) => {
if (form) {
params = form;
} else {
- throw redirect(300, '/');
+ throw redirect(301, '/');
}
}
if (!params.query.trim()) {
- throw redirect(300, '/');
+ throw redirect(301, '/');
}
const { data } = api.search(
diff --git a/frontend/src/routes/search/+page.svelte b/frontend/src/routes/search/+page.svelte
index 9d161e99..66d2d873 100644
--- a/frontend/src/routes/search/+page.svelte
+++ b/frontend/src/routes/search/+page.svelte
@@ -62,7 +62,7 @@
};
// NOTE: save the search query to be used in the header
- $: searchQueryStore.set($page.url.search);
+ $: searchQueryStore?.set($page.url.search);
$: {
results;
@@ -96,9 +96,9 @@
] as const;
const rankSite = (site: DisplayedWebpage, ranking: Ranking) => () => {
- siteRankingsStore.update(($rankings) => ({
+ siteRankingsStore?.update(($rankings) => ({
...$rankings,
- [site.host]: $rankings[site.host] == ranking ? void 0 : ranking,
+ [site.domain]: $rankings[site.domain] == ranking ? void 0 : ranking,
}));
};
@@ -119,13 +119,13 @@
>
<div>
<h2 class="w-fit text-center">
- Do you like results from {modal.site.host}?
+ Do you like results from {modal.site.domain}?
</h2>
<div class="flex justify-center space-x-1.5 pt-2">
{#each rankingChoices as { ranking, kind, Icon }}
<Button
{kind}
- pale={$siteRankingsStore[modal.site.host] != ranking}
+ pale={$siteRankingsStore[modal.site.domain] != ranking}
padding={false}
form="searchbar-form"
on:click={rankSite(modal.site, ranking)}
@@ -153,7 +153,7 @@
<div class="mx-auto flex w-full justify-between">
<div class="flex h-full flex-col justify-center space-x-2 text-sm text-neutral">
<p class="h-fit">
- {#if results.numHits != null }
+ {#if results.numHits != null}
Found {results.numHits.toLocaleString()} results in {(
(results.searchDurationMs ?? 0) / 1000
).toFixed(2)}
diff --git a/frontend/src/routes/search/Result.svelte b/frontend/src/routes/search/Result.svelte
index 9b27fd4b..d74da549 100644
--- a/frontend/src/routes/search/Result.svelte
+++ b/frontend/src/routes/search/Result.svelte
@@ -57,9 +57,9 @@
<div class="inline">
<span id="snippet-text" class="snippet-text">
{#if webpage.snippet.date}
- <span class="text-neutral">
- {webpage.snippet.date} -
- </span>
+ <span class="text-neutral">
+ {webpage.snippet.date} -
+ </span>
{/if}
<span>
<TextSnippet snippet={webpage.snippet.text} />
@@ -70,7 +70,10 @@
</div>
{:else if webpage.snippet.type == 'stackOverflowQA'}
<div class="snippet">
- <StackOverflowSnippet question={webpage.snippet.question} answers={webpage.snippet.answers} />
+ <StackOverflowSnippet
+ question={webpage.snippet.question}
+ answers={webpage.snippet.answers}
+ />
</div>
{/if}
</div>
diff --git a/frontend/src/routes/search/StackOverflowSnippet.svelte b/frontend/src/routes/search/StackOverflowSnippet.svelte
index 11132d40..e9a0fef0 100644
--- a/frontend/src/routes/search/StackOverflowSnippet.svelte
+++ b/frontend/src/routes/search/StackOverflowSnippet.svelte
@@ -1,52 +1,55 @@
-
<script lang="ts">
- import type { StackOverflowQuestion, StackOverflowAnswer } from "$lib/api";
+ import type { StackOverflowQuestion, StackOverflowAnswer } from '$lib/api';
import Code from '$lib/components/Code.svelte';
import HandThumbUp from '~icons/heroicons/hand-thumb-up';
import Check from '~icons/heroicons/check';
- export let question: StackOverflowQuestion;
- export let answers: StackOverflowAnswer[];
+ export let question: StackOverflowQuestion;
+ export let answers: StackOverflowAnswer[];
</script>
+
<div class="line-clamp-2">
-{#each question.body as passage}
- {#if passage.type == "text"}
- {passage.value}
+ {#each question.body as passage}
+ {#if passage.type == 'text'}
+ {passage.value}
{/if}
-{/each}
+ {/each}
</div>
<div class="flex space-x-4 pt-2">
- {#each answers.slice(0, 3) as answer}
+ {#each answers.slice(0, 3) as answer}
<div class="w-1/3 overflow-hidden">
- <a class="block h-56 hover:bg-base-200/80 p-2 border overflow-hidden rounded-lg" href={answer.url}>
- <div class="flex w-full items-center justify-between space-x-1 mb-1 text-neutral text-xs">
- <div class="flex">
- {answer.date}
- </div>
- <div class="flex space-x-1">
- <span class="h-fit">
- {answer.upvotes}
- </span>
- <div class="h-fit">
- <HandThumbUp class="w-4" />
- </div>
- {#if answer.accepted}
- <div class="h-fit text-green-600">
- <Check class="w-4" />
- </div>
- {/if}
+ <a
+ class="block h-56 overflow-hidden rounded-lg border p-2 hover:bg-base-200/80"
+ href={answer.url}
+ >
+ <div class="mb-1 flex w-full items-center justify-between space-x-1 text-xs text-neutral">
+ <div class="flex">
+ {answer.date}
+ </div>
+ <div class="flex space-x-1">
+ <span class="h-fit">
+ {answer.upvotes}
+ </span>
+ <div class="h-fit">
+ <HandThumbUp class="w-4" />
</div>
+ {#if answer.accepted}
+ <div class="h-fit text-green-600">
+ <Check class="w-4" />
+ </div>
+ {/if}
+ </div>
</div>
<div>
- {#each answer.body as passage}
- {#if passage.type == "text"}
- {passage.value}
- {:else if passage.type == "code"}
- <Code code={passage.value} transparentBackground={true} />
- {/if}
- {/each}
+ {#each answer.body as passage}
+ {#if passage.type == 'text'}
+ {passage.value}
+ {:else if passage.type == 'code'}
+ <Code code={passage.value} transparentBackground={true} />
+ {/if}
+ {/each}
</div>
- </a>
+ </a>
</div>
- {/each}
-</div>
\ No newline at end of file
+ {/each}
+</div>
diff --git a/frontend/src/routes/search/Widget.svelte b/frontend/src/routes/search/Widget.svelte
index 97986684..4d2ef854 100644
--- a/frontend/src/routes/search/Widget.svelte
+++ b/frontend/src/routes/search/Widget.svelte
@@ -5,18 +5,17 @@
export let widget: Widget;
</script>
-
{#if widget.type == 'calculator'}
-<div class="rounded-xl border p-5">
- <div class="flex flex-col items-end">
- <div class="flex w-fit text-xs text-neutral">
- {widget.value.input} =
- </div>
- <div class="flex w-fit text-3xl font-bold">
- {widget.value.result}
+ <div class="rounded-xl border p-5">
+ <div class="flex flex-col items-end">
+ <div class="flex w-fit text-xs text-neutral">
+ {widget.value.input} =
+ </div>
+ <div class="flex w-fit text-3xl font-bold">
+ {widget.value.result}
+ </div>
</div>
</div>
-</div>
-{:else if widget.type == "thesaurus"}
-<ThesaurusWidget widget={widget.value} />
+{:else if widget.type == 'thesaurus'}
+ <ThesaurusWidget widget={widget.value} />
{/if}
diff --git a/frontend/src/routes/webmasters/README.md b/frontend/src/routes/webmasters/README.md
index ccd562ac..9ffd8a17 100644
--- a/frontend/src/routes/webmasters/README.md
+++ b/frontend/src/routes/webmasters/README.md
@@ -7,7 +7,7 @@ The crawler uses the user agent `Mozilla/5.0 (compatible; StractBot/0.1; open so
## Politeness
StractBot is a polite crawler. It respects the [robots.txt](https://en.wikipedia.org/wiki/Robots.txt) file of the website it is crawling and tries to not overload the server.
-It does this by waiting a certain amount of time between requests to the same domain. The waiting time is calculated by *min(max(fetchtime \* politeness, 5 sec), 60 sec)* where *fetchtime* is the time it took to fetch the page. The crawler will wait at least 5 sec between requests and at most 60 seconds.
+It does this by waiting a certain amount of time between requests to the same domain. The waiting time is calculated by _min(max(fetchtime \* politeness, 5 sec), 60 sec)_ where _fetchtime_ is the time it took to fetch the page. The crawler will wait at least 5 sec between requests and at most 60 seconds.
This dynamic waiting time tries to prevent us from disrupting servers that cannot handle the load, while not giving unnecessary politeness to servers that can. The politeness factor starts at 1 and is doubled every time the crawler gets a 429 response from the server (to at most 2048).
The crawler looks for the token `StractBot` in the robots.txt file to determine which pages (if any) the crawler is allowed to crawl.
diff --git a/ranking-annotation/.gitignore b/ranking-annotation/.gitignore
new file mode 100644
index 00000000..6635cf55
--- /dev/null
+++ b/ranking-annotation/.gitignore
@@ -0,0 +1,10 @@
+.DS_Store
+node_modules
+/build
+/.svelte-kit
+/package
+.env
+.env.*
+!.env.example
+vite.config.js.timestamp-*
+vite.config.ts.timestamp-*
diff --git a/ranking-annotation/.npmrc b/ranking-annotation/.npmrc
new file mode 100644
index 00000000..b6f27f13
--- /dev/null
+++ b/ranking-annotation/.npmrc
@@ -0,0 +1 @@
+engine-strict=true
diff --git a/ranking-annotation/README.md b/ranking-annotation/README.md
new file mode 100644
index 00000000..5c91169b
--- /dev/null
+++ b/ranking-annotation/README.md
@@ -0,0 +1,38 @@
+# create-svelte
+
+Everything you need to build a Svelte project, powered by [`create-svelte`](https://github.com/sveltejs/kit/tree/master/packages/create-svelte).
+
+## Creating a project
+
+If you're seeing this, you've probably already done this step. Congrats!
+
+```bash
+# create a new project in the current directory
+npm create svelte@latest
+
+# create a new project in my-app
+npm create svelte@latest my-app
+```
+
+## Developing
+
+Once you've created a project and installed dependencies with `npm install` (or `pnpm install` or `yarn`), start a development server:
+
+```bash
+npm run dev
+
+# or start the server and open the app in a new browser tab
+npm run dev -- --open
+```
+
+## Building
+
+To create a production version of your app:
+
+```bash
+npm run build
+```
+
+You can preview the production build with `npm run preview`.
+
+> To deploy your app, you may need to install an [adapter](https://kit.svelte.dev/docs/adapters) for your target environment.
diff --git a/ranking-annotation/package.json b/ranking-annotation/package.json
new file mode 100644
index 00000000..1154e510
--- /dev/null
+++ b/ranking-annotation/package.json
@@ -0,0 +1,41 @@
+{
+ "name": "ranking-annotation",
+ "version": "0.0.1",
+ "private": true,
+ "scripts": {
+ "dev": "vite dev --port 8000",
+ "build": "vite build",
+ "preview": "vite preview",
+ "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
+ "check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch",
+ "lint": "prettier --plugin prettier-plugin-svelte --plugin prettier-plugin-tailwindcss --check . && eslint .",
+ "format": "prettier --plugin prettier-plugin-svelte --plugin prettier-plugin-tailwindcss --write ."
+ },
+ "devDependencies": {
+ "@sveltejs/adapter-auto": "^2.0.0",
+ "@sveltejs/kit": "^1.20.4",
+ "@types/better-sqlite3": "^7.6.6",
+ "@types/papaparse": "^5.3.10",
+ "@typescript-eslint/eslint-plugin": "^6.6.0",
+ "autoprefixer": "^10.4.16",
+ "eslint": "^8.48.0",
+ "eslint-config-prettier": "^9.0.0",
+ "eslint-plugin-svelte": "^2.33.0",
+ "postcss": "^8.4.31",
+ "prettier": "^3.0.3",
+ "prettier-plugin-svelte": "^3.0.3",
+ "prettier-plugin-tailwindcss": "^0.5.4",
+ "svelte": "^4.0.5",
+ "svelte-check": "^3.4.3",
+ "tailwindcss": "^3.3.5",
+ "tslib": "^2.4.1",
+ "typescript": "^5.0.0",
+ "vite": "^4.4.2"
+ },
+ "dependencies": {
+ "better-sqlite3": "^9.0.0",
+ "cassandra-driver": "^4.7.2",
+ "papaparse": "^5.4.1"
+ },
+ "type": "module"
+}
diff --git a/ranking-annotation/postcss.config.js b/ranking-annotation/postcss.config.js
new file mode 100644
index 00000000..2aa7205d
--- /dev/null
+++ b/ranking-annotation/postcss.config.js
@@ -0,0 +1,6 @@
+export default {
+ plugins: {
+ tailwindcss: {},
+ autoprefixer: {},
+ },
+};
diff --git a/ranking-annotation/src/app.css b/ranking-annotation/src/app.css
new file mode 100644
index 00000000..b5c61c95
--- /dev/null
+++ b/ranking-annotation/src/app.css
@@ -0,0 +1,3 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;
diff --git a/ranking-annotation/src/app.d.ts b/ranking-annotation/src/app.d.ts
new file mode 100644
index 00000000..899c7e8f
--- /dev/null
+++ b/ranking-annotation/src/app.d.ts
@@ -0,0 +1,12 @@
+// See https://kit.svelte.dev/docs/types#app
+// for information about these interfaces
+declare global {
+ namespace App {
+ // interface Error {}
+ // interface Locals {}
+ // interface PageData {}
+ // interface Platform {}
+ }
+}
+
+export {};
diff --git a/ranking-annotation/src/app.html b/ranking-annotation/src/app.html
new file mode 100644
index 00000000..84ffad16
--- /dev/null
+++ b/ranking-annotation/src/app.html
@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="en">
+ <head>
+ <meta charset="utf-8" />
+ <link rel="icon" href="%sveltekit.assets%/favicon.png" />
+ <meta name="viewport" content="width=device-width, initial-scale=1" />
+ %sveltekit.head%
+ </head>
+ <body data-sveltekit-preload-data="hover">
+ <div style="display: contents">%sveltekit.body%</div>
+ </body>
+</html>
diff --git a/ranking-annotation/src/lib/dataset.ts b/ranking-annotation/src/lib/dataset.ts
new file mode 100644
index 00000000..9881d2c4
--- /dev/null
+++ b/ranking-annotation/src/lib/dataset.ts
@@ -0,0 +1,30 @@
+import Papa from "papaparse";
+import { readFileSync } from "fs";
+import path from "path";
+
+type Data = string[];
+
+let dataset: Data | undefined = undefined;
+
+export function getDataset(): Data | undefined {
+ if (dataset) {
+ return dataset;
+ }
+
+ const csvFilePath = path.resolve("../data/queries_us.csv");
+ const csvFile = readFileSync(csvFilePath, "utf-8");
+
+ Papa.parse(csvFile, {
+ header: false,
+ delimiter: "\n",
+ complete: (result) => {
+ dataset = result.data.map((row) => {
+ return (row as string[])[0];
+ });
+ },
+ });
+
+ dataset = dataset!.sort(() => Math.random() - 0.5);
+
+ return dataset;
+}
diff --git a/ranking-annotation/src/lib/db.ts b/ranking-annotation/src/lib/db.ts
new file mode 100644
index 00000000..bc7d04ad
--- /dev/null
+++ b/ranking-annotation/src/lib/db.ts
@@ -0,0 +1,109 @@
+import Database from "better-sqlite3";
+import { getDataset } from "./dataset";
+import type { SimpleWebpage } from "./webpage";
+
+const db = new Database("../data/ranking-annotation.sqlite", {
+ fileMustExist: false,
+});
+
+setupDB();
+
+function setupDB() {
+ db.exec(`
+ CREATE TABLE IF NOT EXISTS queries (
+ qid UUID PRIMARY KEY DEFAULT (HEX(RANDOMBLOB(16))),
+ query TEXT NOT NULL UNIQUE
+ );
+ `);
+
+ db.exec(`
+ CREATE TABLE IF NOT EXISTS search_results (
+ qid UUID NOT NULL,
+ url TEXT NOT NULL,
+ orig_rank INTEGER NOT NULL,
+ webpage_json TEXT NOT NULL,
+ anotated_rank INTEGER,
+ PRIMARY KEY (qid, url)
+ );
+ `);
+
+ const data = getDataset()!;
+
+ const insertQuery = db.prepare(`
+ INSERT OR IGNORE INTO queries (query)
+ VALUES (@query)
+ `);
+
+ for (const query of data) {
+ insertQuery.run({ query });
+ }
+
+ return db;
+}
+
+export type Query = {
+ qid: string;
+ query: string;
+ annotated: boolean;
+};
+
+export function getQueries(): Query[] {
+ const query = db.prepare(`
+ SELECT qid, query, EXISTS (
+ SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.anotated_rank IS NOT NULL
+ ) AS annotated
+ FROM queries
+ `);
+
+ return query.all() as Query[];
+}
+
+export type SearchResult = {
+ origRank: number;
+ webpage: SimpleWebpage;
+ annotatedRank: number | null;
+};
+
+export function getSearchResults(qid: string): SearchResult[] {
+ const queryResults = db.prepare(`
+ SELECT url, orig_rank as origRank, webpage_json, anotated_rank AS annotatedRank
+ FROM search_results
+ WHERE qid = @qid
+ `);
+
+ const res = queryResults.all({ qid });
+
+ return res.map((r: any) => ({
+ origRank: r.origRank,
+ annotatedRank: r.annotatedRank,
+ webpage: JSON.parse(r.webpage_json),
+ }));
+}
+
+export const saveSearchResults = (qid: string, searchResults: SearchResult[]) => {
+ const insertSearchResults = db.prepare(`
+ INSERT INTO search_results (qid, url, orig_rank, webpage_json)
+ VALUES (@qid, @url, @origRank, @webpageJson)
+ `);
+
+ for (const searchResult of searchResults) {
+ insertSearchResults.run({
+ qid,
+ url: searchResult.webpage.url,
+ origRank: searchResult.origRank,
+ webpageJson: JSON.stringify(searchResult.webpage),
+ });
+ }
+}
+
+export function getQuery(qid: String): Query {
+ const query = db.prepare(`
+ SELECT qid, query, EXISTS (
+ SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.anotated_rank IS NOT NULL
+ ) AS annotated
+ FROM queries
+ WHERE qid = @qid
+ `);
+
+ return query.get({ qid }) as Query;
+}
diff --git a/ranking-annotation/src/lib/index.ts b/ranking-annotation/src/lib/index.ts
new file mode 100644
index 00000000..856f2b6c
--- /dev/null
+++ b/ranking-annotation/src/lib/index.ts
@@ -0,0 +1 @@
+// place files you want to import through the `$lib` alias in this folder.
diff --git a/ranking-annotation/src/lib/webpage.ts b/ranking-annotation/src/lib/webpage.ts
new file mode 100644
index 00000000..803aa57b
--- /dev/null
+++ b/ranking-annotation/src/lib/webpage.ts
@@ -0,0 +1,70 @@
+export type SimpleWebpage = {
+ title: string;
+ url: string;
+ snippet: string,
+ rankingSignals: Record<string, number>;
+};
+
+export const asSimpleWebpage = (webpage: Webpage): SimpleWebpage => {
+ return {
+ title: webpage.title,
+ url: webpage.url,
+ snippet: webpage.snippet.type === "normal" ? webpage.snippet.text.fragments.map(f => f.text).join("") : webpage.snippet.question.body.map(f => f.value).join(""),
+ rankingSignals: Object.fromEntries(Object.entries(webpage.rankingSignals).map(([key, value]) => [key, value.value])),
+ };
+}
+
+
+export type Webpage = {
+ title: string;
+ url: string;
+ snippet: Snippet;
+ rankingSignals: RankingSignals;
+};
+
+export type RankingSignals = Record<string, RankingSignal>;
+
+export type RankingSignal = {
+ coefficient: number;
+ value: number;
+};
+
+export type Snippet =
+ | {
+ date?: string;
+ text: TextSnippet;
+ type: "normal";
+ }
+ | {
+ answers: StackOverflowAnswer[];
+ question: StackOverflowQuestion;
+ type: "stackOverflowQA";
+ };
+export type StackOverflowAnswer = {
+ accepted: boolean;
+ body: CodeOrText[];
+ date: string;
+ upvotes: number;
+ url: string;
+};
+export type StackOverflowQuestion = {
+ body: CodeOrText[];
+};
+export type TextSnippet = {
+ fragments: TextSnippetFragment[];
+};
+export type TextSnippetFragment = {
+ kind: TextSnippetFragmentKind;
+ text: string;
+};
+export type TextSnippetFragmentKind = "normal" | "highlighted";
+
+export type CodeOrText =
+ | {
+ type: "code";
+ value: string;
+ }
+ | {
+ type: "text";
+ value: string;
+ };
\ No newline at end of file
diff --git a/ranking-annotation/src/routes/+layout.svelte b/ranking-annotation/src/routes/+layout.svelte
new file mode 100644
index 00000000..881d2ba8
--- /dev/null
+++ b/ranking-annotation/src/routes/+layout.svelte
@@ -0,0 +1,5 @@
+<script>
+ import "../app.css";
+</script>
+
+<slot />
diff --git a/ranking-annotation/src/routes/+page.server.ts b/ranking-annotation/src/routes/+page.server.ts
new file mode 100644
index 00000000..99256f7a
--- /dev/null
+++ b/ranking-annotation/src/routes/+page.server.ts
@@ -0,0 +1,7 @@
+import { getQueries } from "$lib/db";
+
+export const load = async () => {
+ const queries = getQueries();
+
+ return { queries };
+};
diff --git a/ranking-annotation/src/routes/+page.svelte b/ranking-annotation/src/routes/+page.svelte
new file mode 100644
index 00000000..e0440221
--- /dev/null
+++ b/ranking-annotation/src/routes/+page.svelte
@@ -0,0 +1,32 @@
+<script lang="ts">
+ import type { PageData } from "./$types";
+ export let data: PageData;
+
+ $: showAnnotated = true;
+ $: queries = showAnnotated
+ ? data.queries
+ : data.queries.filter((q) => !q.annotated);
+</script>
+
+<div class="flex w-full justify-center">
+ <div class="flex max-w-xl grow flex-col space-y-4 pt-2">
+ <h1 class="text-center text-3xl">Ranking Annotations</h1>
+ <label>
+ <input type="checkbox" bind:checked={showAnnotated} class="mr-2" />
+ Show annotated queries
+ </label>
+ <div class="grid grid-rows-1 space-y-2">
+ {#each queries as { qid, query, annotated }}
+ <a href="/annotate/{qid}">
+ <div
+ class="{annotated
+ ? 'bg-green-100'
+ : 'bg-slate-100'} rounded-lg border p-4"
+ >
+ {query}
+ </div>
+ </a>
+ {/each}
+ </div>
+ </div>
+</div>
diff --git a/ranking-annotation/src/routes/annotate/[slug]/+page.server.ts b/ranking-annotation/src/routes/annotate/[slug]/+page.server.ts
new file mode 100644
index 00000000..06837f7f
--- /dev/null
+++ b/ranking-annotation/src/routes/annotate/[slug]/+page.server.ts
@@ -0,0 +1,57 @@
+import { getQuery, getSearchResults, saveSearchResults } from "$lib/db";
+import type { Webpage } from "$lib/webpage";
+import { asSimpleWebpage } from "$lib/webpage";
+
+const search = async (query: string) => {
+ return await fetch(`https://trystract.com/beta/api/search`, {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ accept: "application/json",
+ },
+ body: JSON.stringify({
+ query,
+ numResults: 100,
+ returnRankingSignals: true,
+ }),
+ })
+ .then((res) => {
+ return res.json();
+ })
+ .then((d) => d["webpages"] as Webpage[]);
+};
+
+export const load = async ({ params }) => {
+ const { slug } = params;
+ const qid = slug;
+
+ const query = getQuery(qid);
+ let searchResults = getSearchResults(query.qid);
+
+ if (searchResults.length === 0) {
+ const webpages = (await search(query.query)).map((w) => asSimpleWebpage(w));
+
+ searchResults = webpages.map((w, i) => ({
+ origRank: i,
+ annotatedRank: null,
+ webpage: w,
+ }));
+
+ saveSearchResults(query.qid, searchResults);
+ }
+
+ // sort results by annotatedRank and then origRank
+ searchResults.sort((a, b) => {
+ if (a.annotatedRank === null && b.annotatedRank === null) {
+ return a.origRank - b.origRank;
+ } else if (a.annotatedRank === null) {
+ return 1;
+ } else if (b.annotatedRank === null) {
+ return -1;
+ } else {
+ return a.annotatedRank - b.annotatedRank;
+ }
+ });
+
+ return { query, searchResults };
+};
\ No newline at end of file
diff --git a/ranking-annotation/src/routes/annotate/[slug]/+page.svelte b/ranking-annotation/src/routes/annotate/[slug]/+page.svelte
new file mode 100644
index 00000000..85a1d3f0
--- /dev/null
+++ b/ranking-annotation/src/routes/annotate/[slug]/+page.svelte
@@ -0,0 +1,28 @@
+<script lang="ts">
+ import type { PageData } from "./$types";
+ export let data: PageData;
+</script>
+
+<div class="flex flex-col w-full items-center">
+ <div class="max-w-xl flex flex-col space-y-5">
+ <div>
+ <b class="font-bold">Query:</b> {data.query.query}
+ </div>
+
+ <div class="flex flex-col space-y-3">
+ {#each data.searchResults as res}
+ <div class="py-2 px-2 border shadow-sm rounded-lg">
+ <div>
+ <a href="{res.webpage.url}">{res.webpage.url}</a>
+ </div>
+ <div>
+ <a href="{res.webpage.url}">{res.webpage.title}</a>
+ </div>
+ <div class="line-clamp-3 text-sm">
+ {res.webpage.snippet}
+ </div>
+ </div>
+ {/each}
+ </div>
+ </div>
+</div>
\ No newline at end of file
diff --git a/ranking-annotation/src/routes/api/data/+server.ts b/ranking-annotation/src/routes/api/data/+server.ts
new file mode 100644
index 00000000..9b71cc80
--- /dev/null
+++ b/ranking-annotation/src/routes/api/data/+server.ts
@@ -0,0 +1,11 @@
+import { getDataset } from "$lib/dataset";
+
+export async function GET({}): Promise<Response> {
+ const data = getDataset();
+
+ if (data) {
+ return new Response(JSON.stringify({ data }));
+ } else {
+ return new Response("Failed to parse CSV data", { status: 500 });
+ }
+}
diff --git a/ranking-annotation/static/favicon.png b/ranking-annotation/static/favicon.png
new file mode 100644
index 00000000..825b9e65
Binary files /dev/null and b/ranking-annotation/static/favicon.png differ
diff --git a/ranking-annotation/svelte.config.js b/ranking-annotation/svelte.config.js
new file mode 100644
index 00000000..5627353e
--- /dev/null
+++ b/ranking-annotation/svelte.config.js
@@ -0,0 +1,18 @@
+import adapter from "@sveltejs/adapter-auto";
+import { vitePreprocess } from "@sveltejs/kit/vite";
+
+/** @type {import('@sveltejs/kit').Config} */
+const config = {
+ // Consult https://kit.svelte.dev/docs/integrations#preprocessors
+ // for more information about preprocessors
+ preprocess: vitePreprocess(),
+
+ kit: {
+ // adapter-auto only supports some environments, see https://kit.svelte.dev/docs/adapter-auto for a list.
+ // If your environment is not supported or you settled on a specific environment, switch out the adapter.
+ // See https://kit.svelte.dev/docs/adapters for more information about adapters.
+ adapter: adapter(),
+ },
+};
+
+export default config;
diff --git a/ranking-annotation/tailwind.config.ts b/ranking-annotation/tailwind.config.ts
new file mode 100644
index 00000000..03114c64
--- /dev/null
+++ b/ranking-annotation/tailwind.config.ts
@@ -0,0 +1,8 @@
+/** @type {import('tailwindcss').Config} */
+export default {
+ content: ["./src/**/*.{html,js,svelte,ts}"],
+ theme: {
+ extend: {},
+ },
+ plugins: [],
+};
diff --git a/ranking-annotation/tsconfig.json b/ranking-annotation/tsconfig.json
new file mode 100644
index 00000000..794b95b6
--- /dev/null
+++ b/ranking-annotation/tsconfig.json
@@ -0,0 +1,17 @@
+{
+ "extends": "./.svelte-kit/tsconfig.json",
+ "compilerOptions": {
+ "allowJs": true,
+ "checkJs": true,
+ "esModuleInterop": true,
+ "forceConsistentCasingInFileNames": true,
+ "resolveJsonModule": true,
+ "skipLibCheck": true,
+ "sourceMap": true,
+ "strict": true
+ }
+ // Path aliases are handled by https://kit.svelte.dev/docs/configuration#alias
+ //
+ // If you want to overwrite includes/excludes, make sure to copy over the relevant includes/excludes
+ // from the referenced tsconfig.json - TypeScript does not merge them in
+}
diff --git a/ranking-annotation/vite.config.ts b/ranking-annotation/vite.config.ts
new file mode 100644
index 00000000..80864b9d
--- /dev/null
+++ b/ranking-annotation/vite.config.ts
@@ -0,0 +1,6 @@
+import { sveltekit } from "@sveltejs/kit/vite";
+import { defineConfig } from "vite";
+
+export default defineConfig({
+ plugins: [sveltekit()],
+});