An open source, non-profit web search engine
Store source in the database
Daoud Clarked9a15b4fd35e
diff --git a/devdata/index-v2.tinysearch b/devdata/index-v2.tinysearchindex 0b29ce2..f73023c 100644Binary files a/devdata/index-v2.tinysearch and b/devdata/index-v2.tinysearch differdiff --git a/front-end/src/components/molecules/validate-button.js b/front-end/src/components/molecules/validate-button.jsindex 997af12..177aabe 100644--- a/front-end/src/components/molecules/validate-button.js+++ b/front-end/src/components/molecules/validate-button.js@@ -27,7 +27,8 @@ export default define('validate-button', class extends HTMLButtonElement {const curationValidateEvent = new CustomEvent('curate-validate-result', {detail: {data: {- validate_index: index+ validate_index: index,+ is_validated: this.isValidated(),}}});diff --git a/front-end/src/components/organisms/results.js b/front-end/src/components/organisms/results.jsindex cfeab6b..3deeed1 100644--- a/front-end/src/components/organisms/results.js+++ b/front-end/src/components/organisms/results.js@@ -156,7 +156,8 @@ class ResultsHandler {url: resultElement.querySelector('a').href,title: resultElement.querySelector('.title').innerText,extract: resultElement.querySelector('.extract').innerText,- curated: resultElement.querySelector('.curate-approve').isValidated()+ validated: resultElement.querySelector('.curate-approve').classList.contains('validated'),+ source: resultElement.querySelector('.source').innerText,}results.push(result);}diff --git a/mwmbl/platform/curate.py b/mwmbl/platform/curate.pyindex 782195f..16ffa86 100644--- a/mwmbl/platform/curate.py+++ b/mwmbl/platform/curate.py@@ -1,5 +1,5 @@from logging import getLogger-from typing import Any+from typing import Any, Optionalfrom urllib.parse import parse_qsfrom ninja import Router, NinjaAPI@@ -8,7 +8,7 @@ from mwmbl.indexer.update_urls import get_datetime_from_timestampfrom mwmbl.models import UserCurationfrom mwmbl.platform.data import CurateBegin, CurateMove, CurateDelete, CurateAdd, CurateValidate, \make_curation_type-from mwmbl.tinysearchengine.indexer import TinyIndex, Document+from mwmbl.tinysearchengine.indexer import TinyIndex, Document, DocumentStatefrom mwmbl.tokenizer import tokenizefrom mwmbl.utils import add_term_info, add_term_infos@@ -67,7 +67,14 @@ def create_router(index_path: str, version: str) -> NinjaAPI:term = " ".join(tokens)documents = [- Document(result.title, result.url, result.extract, MAX_CURATED_SCORE - i, term, result.curated)+ Document(+ title=result.title,+ url=result.url,+ extract=result.extract,+ score=MAX_CURATED_SCORE - i,+ term=term,+ state=_get_document_state(result.validated, result.source),+ )for i, result in enumerate(curation.results)]@@ -87,3 +94,12 @@ def create_router(index_path: str, version: str) -> NinjaAPI:return router+def _get_document_state(validated: bool, source: str) -> Optional[DocumentState]:+ if validated:+ return DocumentState.VALIDATED+ elif source.lower() == "user":+ return DocumentState.FROM_USER+ elif source.lower() == "google":+ return DocumentState.FROM_GOOGLE+ else:+ return DocumentState.CURATEDdiff --git a/mwmbl/platform/data.py b/mwmbl/platform/data.pyindex 93b8f41..5929194 100644--- a/mwmbl/platform/data.py+++ b/mwmbl/platform/data.py@@ -8,7 +8,8 @@ class Result(Schema):url: strtitle: strextract: str- curated: bool+ validated: bool+ source: strclass CurateBegin(Schema):diff --git a/mwmbl/templates/result.html b/mwmbl/templates/result.htmlindex 339d293..c63b8fb 100644--- a/mwmbl/templates/result.html+++ b/mwmbl/templates/result.html@@ -3,7 +3,7 @@<div class="result-container"><div class="result-link"><a href="{{result.url}}">- <p><span class='link'>{{result.url}}</span> <span class="via">— found via {{result.source|title}}</span></p>+ <p><span class='link'>{{result.url}}</span> <span class="via">— found via <span class="source">{{result.source|title}}</span></span></p><p class='title'>{{result.title|strengthen}}</p></a><p class='extract'>{{result.extract|strengthen}}</p>diff --git a/mwmbl/tinysearchengine/indexer.py b/mwmbl/tinysearchengine/indexer.pyindex 33b0d02..156c519 100644--- a/mwmbl/tinysearchengine/indexer.py+++ b/mwmbl/tinysearchengine/indexer.py@@ -31,6 +31,9 @@ def astuple(dc):class DocumentState(IntEnum):+ """+ The state of the document in the index. A value of None indicates a document from our normal crawling procedure.+ """CURATED = 0VALIDATED = 1FROM_USER = 2diff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.pyindex 9ccec5d..4ca4505 100644--- a/mwmbl/tinysearchengine/rank.py+++ b/mwmbl/tinysearchengine/rank.py@@ -175,7 +175,7 @@ class Ranker:# Check for curationcuration_term = " ".join(terms)curation_items = self.tiny_index.retrieve(curation_term)- curated_items = [d for d in curation_items if d.state in {DocumentState.CURATED, DocumentState.VALIDATED}+ curated_items = [d for d in curation_items if d.state is not Noneand d.term == curation_term]if len(curated_items) > 0: