web search done right
ranking annotation: labels as scores instead of ranks. change the labels in the ranking annotation tool to be scores from 0-4 (inclusive) where a higher score is a more relevant result. this allows multiple results to have the same score if they are all relevant to the query, whereas having rank labels means only a single result can get a specific label for each query.
Mikkel Denker155b5f337d03
diff --git a/ltr/lambdamart.py b/ltr/lambdamart.pyindex 9f727c87..33792c94 100644--- a/ltr/lambdamart.py+++ b/ltr/lambdamart.py@@ -13,7 +13,7 @@ res = cur.execute(SELECT qid, queryFROM queriesWHERE EXISTS (- SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotated_rank IS NOT NULL+ SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotation IS NOT NULL)""")@@ -22,24 +22,23 @@ queries = {qid: {"query": query} for qid, query in res.fetchall()}for qid in queries:res = cur.execute("""- SELECT qid, url, annotated_rank, webpage_json+ SELECT qid, url, annotation, webpage_jsonFROM search_resultsWHERE qid = ?""",(qid,),)urls = {- url: {"rank": rank, "signals": json.loads(page)["rankingSignals"]}- for _, url, rank, page in res.fetchall()+ url: {"label": label, "signals": json.loads(page)["rankingSignals"]}+ for _, url, label, page in res.fetchall()}urls = [- (url, w["rank"], w["signals"])+ (url, w["label"], w["signals"])for url, w in urls.items()- if w["rank"] is not None+ if w["label"] is not None]- urls = sorted(urls, key=lambda x: x[1], reverse=False)- urls = [(url, i, signals) for (url, _, signals), i in zip(urls, range(len(urls)))]+ urls = sorted(urls, key=lambda x: x[1], reverse=True)queries[qid]["urls"] = urlsdiff --git a/tools/ranking-annotation/package.json b/tools/ranking-annotation/package.jsonindex 66e6ef62..59a94500 100644--- a/tools/ranking-annotation/package.json+++ b/tools/ranking-annotation/package.json@@ -39,8 +39,7 @@"dependencies": {"better-sqlite3": "^9.0.0","cassandra-driver": "^4.7.2",- "papaparse": "^5.4.1",- "svelte-dnd-action": "^0.9.31"+ "papaparse": "^5.4.1"},"type": "module"-}+}\ No newline at end of filediff --git a/tools/ranking-annotation/src/lib/db.ts b/tools/ranking-annotation/src/lib/db.tsindex 446df6e4..62911aa2 100644--- a/tools/ranking-annotation/src/lib/db.ts+++ b/tools/ranking-annotation/src/lib/db.ts@@ -22,7 +22,7 @@ function setupDB() {url TEXT NOT NULL,orig_rank INTEGER NOT NULL,webpage_json TEXT NOT NULL,- annotated_rank INTEGER,+ annotation INTEGER,PRIMARY KEY (qid, url));`);@@ -50,7 +50,7 @@ export type Query = {export function getQueries(): Query[] {const query = db.prepare(`SELECT qid, query, EXISTS (- SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotated_rank IS NOT NULL+ SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotation IS NOT NULL) AS annotatedFROM queries ORDER BY qid`);@@ -61,7 +61,7 @@ export function getQueries(): Query[] {export function getNextQuery(qid: string): Query | undefined {const query = db.prepare(`SELECT qid, query, EXISTS (- SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotated_rank IS NOT NULL+ SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotation IS NOT NULL) AS annotatedFROM queriesWHERE qid > @qid@@ -77,7 +77,7 @@ export function getNextQuery(qid: string): Query | undefined {export function getPreviousQuery(qid: string): Query | undefined {const query = db.prepare(`SELECT qid, query, EXISTS (- SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotated_rank IS NOT NULL+ SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotation IS NOT NULL) AS annotatedFROM queriesWHERE qid < @qid@@ -94,12 +94,12 @@ export type SearchResult = {id: string;origRank: number;webpage: SimpleWebpage;- annotatedRank: number | null;+ annotation: number | null;};export function getSearchResults(qid: string): SearchResult[] {const queryResults = db.prepare(`- SELECT url, orig_rank as origRank, webpage_json, annotated_rank AS annotatedRank+ SELECT url, orig_rank as origRank, webpage_json, annotationFROM search_resultsWHERE qid = @qid`);@@ -109,17 +109,17 @@ export function getSearchResults(qid: string): SearchResult[] {return res.map((r: any) => ({id: `${qid}-${r.url}`,origRank: r.origRank,- annotatedRank: r.annotatedRank,+ annotation: r.annotation,webpage: JSON.parse(r.webpage_json),}));}export const saveSearchResults = (qid: string, searchResults: SearchResult[]) => {const upsertSearchResults = db.prepare(`- INSERT INTO search_results (qid, url, orig_rank, webpage_json, annotated_rank)- VALUES (@qid, @url, @origRank, @webpageJson, @annotatedRank)+ INSERT INTO search_results (qid, url, orig_rank, webpage_json, annotation)+ VALUES (@qid, @url, @origRank, @webpageJson, @annotation)ON CONFLICT(qid, url) DO UPDATE SET- annotated_rank = excluded.annotated_rank+ annotation = excluded.annotation`);for (const searchResult of searchResults) {@@ -128,7 +128,7 @@ export const saveSearchResults = (qid: string, searchResults: SearchResult[]) =>url: searchResult.webpage.url,origRank: searchResult.origRank,webpageJson: JSON.stringify(searchResult.webpage),- annotatedRank: searchResult.annotatedRank,+ annotation: searchResult.annotation,});}}@@ -136,7 +136,7 @@ export const saveSearchResults = (qid: string, searchResults: SearchResult[]) =>export function getQuery(qid: String): Query | undefined {const query = db.prepare(`SELECT qid, query, EXISTS (- SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotated_rank IS NOT NULL+ SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotation IS NOT NULL) AS annotatedFROM queriesWHERE qid = @qiddiff --git a/tools/ranking-annotation/src/routes/annotate/[slug]/+page.server.ts b/tools/ranking-annotation/src/routes/annotate/[slug]/+page.server.tsindex f7fc2492..4654e3a1 100644--- a/tools/ranking-annotation/src/routes/annotate/[slug]/+page.server.ts+++ b/tools/ranking-annotation/src/routes/annotate/[slug]/+page.server.ts@@ -43,23 +43,24 @@ export const load = async ({ params }) => {searchResults = webpages.map((w, i) => ({id: `${query.qid}-${w.url}`,origRank: i,- annotatedRank: null,+ annotation: null,webpage: w,}));saveSearchResults(query.qid, searchResults);}- // sort results by annotatedRank and then origRank+ // sort results by annotation and then origRank+ // annotation is 0-4 where 4 is best, null is unannotatedsearchResults.sort((a, b) => {- if (a.annotatedRank === null && b.annotatedRank === null) {+ if ((a.annotation === null && b.annotation === null) || a.annotation === b.annotation) {return a.origRank - b.origRank;- } else if (a.annotatedRank === null) {+ } else if (a.annotation === null) {return 1;- } else if (b.annotatedRank === null) {+ } else if (b.annotation === null) {return -1;} else {- return a.annotatedRank - b.annotatedRank;+ return b.annotation - a.annotation;}});diff --git a/tools/ranking-annotation/src/routes/annotate/[slug]/+page.svelte b/tools/ranking-annotation/src/routes/annotate/[slug]/+page.svelteindex 1a103788..ba30cac5 100644--- a/tools/ranking-annotation/src/routes/annotate/[slug]/+page.svelte+++ b/tools/ranking-annotation/src/routes/annotate/[slug]/+page.svelte@@ -1,33 +1,13 @@<script lang="ts">- import Check from '~icons/heroicons/check-20-solid';import Home from '~icons/heroicons/home-20-solid';import ArrowLeft from '~icons/heroicons/arrow-left-20-solid';import ArrowRight from '~icons/heroicons/arrow-right-20-solid';- import { dndzone } from 'svelte-dnd-action';import type { PageData } from "./$types";- import type { SearchResult } from '$lib/db';- import { browser } from '$app/environment';+ import { browser } from '$app/environment';$: shownSignals = -1;$: searchResults = data.searchResults;-- function updateRanks() {- searchResults.forEach((res, i) => {- if (res.annotatedRank != null) {- res.annotatedRank = i;- }- });- }-- function handleConsider(event: CustomEvent<{ items: SearchResult[] }>) {- searchResults = event.detail.items;- }-- async function handleDrop(event: CustomEvent<{ items: SearchResult[] }>) {- searchResults = event.detail.items;- updateRanks();- await save();- }+ $: selected = 0;async function save() {if (!browser) {@@ -46,6 +26,45 @@})}+ function handleKeyPress(event: KeyboardEvent) {+ if (event.key === "ArrowUp") {+ event.preventDefault();+ selected = Math.max(0, selected - 1);+ } else if (event.key === "ArrowDown") {+ event.preventDefault();+ selected = Math.min(searchResults.length - 1, selected + 1);+ }++ // scroll to selected+ const el = document.getElementById(searchResults[selected].id);+ el!.scrollIntoView({+ block: "nearest",+ inline: "nearest",+ });++ // if key is 0-4, set annotation+ if (event.key >= "0" && event.key <= "4") {+ searchResults[selected].annotation = parseInt(event.key);+ save();+ } else if (event.key === "§") {+ searchResults[selected].annotation = 0;+ save();+ } else if (event.key === "Backspace" || event.key === "-") {+ searchResults[selected].annotation = null;+ save();+ }++ // go to next/previous query+ if (event.key === "ArrowLeft") {+ selected = 0;+ window.location.href = `/annotate/${data.previousQuery?.qid}`;+ } else if (event.key === "ArrowRight") {+ selected = 0;+ window.location.href = `/annotate/${data.nextQuery?.qid}`;+ }++ }+$: searchResults && save();export let data: PageData;@@ -73,24 +92,27 @@</a></div></div>- <section use:dndzone={{ items: searchResults, dropTargetStyle: {} }} on:consider={handleConsider} on:finalize={handleDrop} class="flex flex-col space-y-3">+ <section class="flex flex-col space-y-3">{#each searchResults as res, i (res.id)}- <div>+ <div id={res.id}><div class="relative">- <button class="absolute left-[-2em] top-3" on:click={() => (res.annotatedRank == null) ? res.annotatedRank = i : res.annotatedRank = null}>- {#if res.annotatedRank != null}- <div class="text-green-500">- <Check class="w-6 h-6" />- </div>- {:else}- <div class="text-slate-200">- <Check class="w-6 h-6" />- </div>- {/if}- </button>- <div class="py-2 px-2 bg-slate-100 border shadow-sm rounded-lg">- <div>+ <!-- svelte-ignore a11y-no-noninteractive-element-interactions -->+ <div role="listitem" class="py-2 px-2 {selected == i ? 'bg-slate-300' : 'bg-slate-100'} border shadow-sm rounded-lg" on:click={() => selected = i} on:keypress={handleKeyPress}>+ <div class="w-full flex justify-between"><a href="{res.webpage.url}" target="_blank">{res.webpage.url}</a>+ {#if res.annotation != null}+ {#if res.annotation == 4}+ <div class="text-green-600">{res.annotation}</div>+ {:else if res.annotation == 3}+ <div class="text-green-500">{res.annotation}</div>+ {:else if res.annotation == 2}+ <div class="text-yellow-500">{res.annotation}</div>+ {:else if res.annotation == 1}+ <div class="text-red-500">{res.annotation}</div>+ {:else if res.annotation == 0}+ <div class="text-red-600">{res.annotation}</div>+ {/if}+ {/if}</div><div class="font-bold"><a href="{res.webpage.url}" target="_blank">{res.webpage.title}</a>@@ -125,4 +147,6 @@{/each}</section></div>-</div>\ No newline at end of file+</div>++<svelte:window on:keydown={handleKeyPress}/>\ No newline at end of file