Skip to content

web search done right

ranking annotation: labels as scores instead of ranks. change the labels in the ranking annotation tool to be scores from 0-4 (inclusive) where a higher score is a more relevant result. this allows multiple results to have the same score if they are all relevant to the query, whereas having rank labels means only a single result can get a specific label for each query.

Mikkel Denker155b5f337d03
diff --git a/ltr/lambdamart.py b/ltr/lambdamart.py
index 9f727c87..33792c94 100644
--- a/ltr/lambdamart.py
+++ b/ltr/lambdamart.py
@@ -13,7 +13,7 @@ res = cur.execute(
SELECT qid, query
FROM queries
WHERE EXISTS (
- SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotated_rank IS NOT NULL
+ SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotation IS NOT NULL
)
"""
)
@@ -22,24 +22,23 @@ queries = {qid: {"query": query} for qid, query in res.fetchall()}
for qid in queries:
res = cur.execute(
"""
- SELECT qid, url, annotated_rank, webpage_json
+ SELECT qid, url, annotation, webpage_json
FROM search_results
WHERE qid = ?
""",
(qid,),
)
urls = {
- url: {"rank": rank, "signals": json.loads(page)["rankingSignals"]}
- for _, url, rank, page in res.fetchall()
+ url: {"label": label, "signals": json.loads(page)["rankingSignals"]}
+ for _, url, label, page in res.fetchall()
}
urls = [
- (url, w["rank"], w["signals"])
+ (url, w["label"], w["signals"])
for url, w in urls.items()
- if w["rank"] is not None
+ if w["label"] is not None
]
- urls = sorted(urls, key=lambda x: x[1], reverse=False)
- urls = [(url, i, signals) for (url, _, signals), i in zip(urls, range(len(urls)))]
+ urls = sorted(urls, key=lambda x: x[1], reverse=True)
queries[qid]["urls"] = urls
diff --git a/tools/ranking-annotation/package.json b/tools/ranking-annotation/package.json
index 66e6ef62..59a94500 100644
--- a/tools/ranking-annotation/package.json
+++ b/tools/ranking-annotation/package.json
@@ -39,8 +39,7 @@
"dependencies": {
"better-sqlite3": "^9.0.0",
"cassandra-driver": "^4.7.2",
- "papaparse": "^5.4.1",
- "svelte-dnd-action": "^0.9.31"
+ "papaparse": "^5.4.1"
},
"type": "module"
-}
+}
\ No newline at end of file
diff --git a/tools/ranking-annotation/src/lib/db.ts b/tools/ranking-annotation/src/lib/db.ts
index 446df6e4..62911aa2 100644
--- a/tools/ranking-annotation/src/lib/db.ts
+++ b/tools/ranking-annotation/src/lib/db.ts
@@ -22,7 +22,7 @@ function setupDB() {
url TEXT NOT NULL,
orig_rank INTEGER NOT NULL,
webpage_json TEXT NOT NULL,
- annotated_rank INTEGER,
+ annotation INTEGER,
PRIMARY KEY (qid, url)
);
`);
@@ -50,7 +50,7 @@ export type Query = {
export function getQueries(): Query[] {
const query = db.prepare(`
SELECT qid, query, EXISTS (
- SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotated_rank IS NOT NULL
+ SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotation IS NOT NULL
) AS annotated
FROM queries ORDER BY qid
`);
@@ -61,7 +61,7 @@ export function getQueries(): Query[] {
export function getNextQuery(qid: string): Query | undefined {
const query = db.prepare(`
SELECT qid, query, EXISTS (
- SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotated_rank IS NOT NULL
+ SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotation IS NOT NULL
) AS annotated
FROM queries
WHERE qid > @qid
@@ -77,7 +77,7 @@ export function getNextQuery(qid: string): Query | undefined {
export function getPreviousQuery(qid: string): Query | undefined {
const query = db.prepare(`
SELECT qid, query, EXISTS (
- SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotated_rank IS NOT NULL
+ SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotation IS NOT NULL
) AS annotated
FROM queries
WHERE qid < @qid
@@ -94,12 +94,12 @@ export type SearchResult = {
id: string;
origRank: number;
webpage: SimpleWebpage;
- annotatedRank: number | null;
+ annotation: number | null;
};
export function getSearchResults(qid: string): SearchResult[] {
const queryResults = db.prepare(`
- SELECT url, orig_rank as origRank, webpage_json, annotated_rank AS annotatedRank
+ SELECT url, orig_rank as origRank, webpage_json, annotation
FROM search_results
WHERE qid = @qid
`);
@@ -109,17 +109,17 @@ export function getSearchResults(qid: string): SearchResult[] {
return res.map((r: any) => ({
id: `${qid}-${r.url}`,
origRank: r.origRank,
- annotatedRank: r.annotatedRank,
+ annotation: r.annotation,
webpage: JSON.parse(r.webpage_json),
}));
}
export const saveSearchResults = (qid: string, searchResults: SearchResult[]) => {
const upsertSearchResults = db.prepare(`
- INSERT INTO search_results (qid, url, orig_rank, webpage_json, annotated_rank)
- VALUES (@qid, @url, @origRank, @webpageJson, @annotatedRank)
+ INSERT INTO search_results (qid, url, orig_rank, webpage_json, annotation)
+ VALUES (@qid, @url, @origRank, @webpageJson, @annotation)
ON CONFLICT(qid, url) DO UPDATE SET
- annotated_rank = excluded.annotated_rank
+ annotation = excluded.annotation
`);
for (const searchResult of searchResults) {
@@ -128,7 +128,7 @@ export const saveSearchResults = (qid: string, searchResults: SearchResult[]) =>
url: searchResult.webpage.url,
origRank: searchResult.origRank,
webpageJson: JSON.stringify(searchResult.webpage),
- annotatedRank: searchResult.annotatedRank,
+ annotation: searchResult.annotation,
});
}
}
@@ -136,7 +136,7 @@ export const saveSearchResults = (qid: string, searchResults: SearchResult[]) =>
export function getQuery(qid: String): Query | undefined {
const query = db.prepare(`
SELECT qid, query, EXISTS (
- SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotated_rank IS NOT NULL
+ SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotation IS NOT NULL
) AS annotated
FROM queries
WHERE qid = @qid
diff --git a/tools/ranking-annotation/src/routes/annotate/[slug]/+page.server.ts b/tools/ranking-annotation/src/routes/annotate/[slug]/+page.server.ts
index f7fc2492..4654e3a1 100644
--- a/tools/ranking-annotation/src/routes/annotate/[slug]/+page.server.ts
+++ b/tools/ranking-annotation/src/routes/annotate/[slug]/+page.server.ts
@@ -43,23 +43,24 @@ export const load = async ({ params }) => {
searchResults = webpages.map((w, i) => ({
id: `${query.qid}-${w.url}`,
origRank: i,
- annotatedRank: null,
+ annotation: null,
webpage: w,
}));
saveSearchResults(query.qid, searchResults);
}
- // sort results by annotatedRank and then origRank
+ // sort results by annotation and then origRank
+ // annotation is 0-4 where 4 is best, null is unannotated
searchResults.sort((a, b) => {
- if (a.annotatedRank === null && b.annotatedRank === null) {
+ if ((a.annotation === null && b.annotation === null) || a.annotation === b.annotation) {
return a.origRank - b.origRank;
- } else if (a.annotatedRank === null) {
+ } else if (a.annotation === null) {
return 1;
- } else if (b.annotatedRank === null) {
+ } else if (b.annotation === null) {
return -1;
} else {
- return a.annotatedRank - b.annotatedRank;
+ return b.annotation - a.annotation;
}
});
diff --git a/tools/ranking-annotation/src/routes/annotate/[slug]/+page.svelte b/tools/ranking-annotation/src/routes/annotate/[slug]/+page.svelte
index 1a103788..ba30cac5 100644
--- a/tools/ranking-annotation/src/routes/annotate/[slug]/+page.svelte
+++ b/tools/ranking-annotation/src/routes/annotate/[slug]/+page.svelte
@@ -1,33 +1,13 @@
<script lang="ts">
- import Check from '~icons/heroicons/check-20-solid';
import Home from '~icons/heroicons/home-20-solid';
import ArrowLeft from '~icons/heroicons/arrow-left-20-solid';
import ArrowRight from '~icons/heroicons/arrow-right-20-solid';
- import { dndzone } from 'svelte-dnd-action';
import type { PageData } from "./$types";
- import type { SearchResult } from '$lib/db';
- import { browser } from '$app/environment';
+ import { browser } from '$app/environment';
$: shownSignals = -1;
$: searchResults = data.searchResults;
-
- function updateRanks() {
- searchResults.forEach((res, i) => {
- if (res.annotatedRank != null) {
- res.annotatedRank = i;
- }
- });
- }
-
- function handleConsider(event: CustomEvent<{ items: SearchResult[] }>) {
- searchResults = event.detail.items;
- }
-
- async function handleDrop(event: CustomEvent<{ items: SearchResult[] }>) {
- searchResults = event.detail.items;
- updateRanks();
- await save();
- }
+ $: selected = 0;
async function save() {
if (!browser) {
@@ -46,6 +26,45 @@
})
}
+ function handleKeyPress(event: KeyboardEvent) {
+ if (event.key === "ArrowUp") {
+ event.preventDefault();
+ selected = Math.max(0, selected - 1);
+ } else if (event.key === "ArrowDown") {
+ event.preventDefault();
+ selected = Math.min(searchResults.length - 1, selected + 1);
+ }
+
+ // scroll to selected
+ const el = document.getElementById(searchResults[selected].id);
+ el!.scrollIntoView({
+ block: "nearest",
+ inline: "nearest",
+ });
+
+ // if key is 0-4, set annotation
+ if (event.key >= "0" && event.key <= "4") {
+ searchResults[selected].annotation = parseInt(event.key);
+ save();
+ } else if (event.key === "§") {
+ searchResults[selected].annotation = 0;
+ save();
+ } else if (event.key === "Backspace" || event.key === "-") {
+ searchResults[selected].annotation = null;
+ save();
+ }
+
+ // go to next/previous query
+ if (event.key === "ArrowLeft") {
+ selected = 0;
+ window.location.href = `/annotate/${data.previousQuery?.qid}`;
+ } else if (event.key === "ArrowRight") {
+ selected = 0;
+ window.location.href = `/annotate/${data.nextQuery?.qid}`;
+ }
+
+ }
+
$: searchResults && save();
export let data: PageData;
@@ -73,24 +92,27 @@
</a>
</div>
</div>
- <section use:dndzone={{ items: searchResults, dropTargetStyle: {} }} on:consider={handleConsider} on:finalize={handleDrop} class="flex flex-col space-y-3">
+ <section class="flex flex-col space-y-3">
{#each searchResults as res, i (res.id)}
- <div>
+ <div id={res.id}>
<div class="relative">
- <button class="absolute left-[-2em] top-3" on:click={() => (res.annotatedRank == null) ? res.annotatedRank = i : res.annotatedRank = null}>
- {#if res.annotatedRank != null}
- <div class="text-green-500">
- <Check class="w-6 h-6" />
- </div>
- {:else}
- <div class="text-slate-200">
- <Check class="w-6 h-6" />
- </div>
- {/if}
- </button>
- <div class="py-2 px-2 bg-slate-100 border shadow-sm rounded-lg">
- <div>
+ <!-- svelte-ignore a11y-no-noninteractive-element-interactions -->
+ <div role="listitem" class="py-2 px-2 {selected == i ? 'bg-slate-300' : 'bg-slate-100'} border shadow-sm rounded-lg" on:click={() => selected = i} on:keypress={handleKeyPress}>
+ <div class="w-full flex justify-between">
<a href="{res.webpage.url}" target="_blank">{res.webpage.url}</a>
+ {#if res.annotation != null}
+ {#if res.annotation == 4}
+ <div class="text-green-600">{res.annotation}</div>
+ {:else if res.annotation == 3}
+ <div class="text-green-500">{res.annotation}</div>
+ {:else if res.annotation == 2}
+ <div class="text-yellow-500">{res.annotation}</div>
+ {:else if res.annotation == 1}
+ <div class="text-red-500">{res.annotation}</div>
+ {:else if res.annotation == 0}
+ <div class="text-red-600">{res.annotation}</div>
+ {/if}
+ {/if}
</div>
<div class="font-bold">
<a href="{res.webpage.url}" target="_blank">{res.webpage.title}</a>
@@ -125,4 +147,6 @@
{/each}
</section>
</div>
-</div>
\ No newline at end of file
+</div>
+
+<svelte:window on:keydown={handleKeyPress}/>
\ No newline at end of file