Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Store source in the database

Daoud Clarked9a15b4fd35e
diff --git a/devdata/index-v2.tinysearch b/devdata/index-v2.tinysearch
index 0b29ce2..f73023c 100644
Binary files a/devdata/index-v2.tinysearch and b/devdata/index-v2.tinysearch differ
diff --git a/front-end/src/components/molecules/validate-button.js b/front-end/src/components/molecules/validate-button.js
index 997af12..177aabe 100644
--- a/front-end/src/components/molecules/validate-button.js
+++ b/front-end/src/components/molecules/validate-button.js
@@ -27,7 +27,8 @@ export default define('validate-button', class extends HTMLButtonElement {
const curationValidateEvent = new CustomEvent('curate-validate-result', {
detail: {
data: {
- validate_index: index
+ validate_index: index,
+ is_validated: this.isValidated(),
}
}
});
diff --git a/front-end/src/components/organisms/results.js b/front-end/src/components/organisms/results.js
index cfeab6b..3deeed1 100644
--- a/front-end/src/components/organisms/results.js
+++ b/front-end/src/components/organisms/results.js
@@ -156,7 +156,8 @@ class ResultsHandler {
url: resultElement.querySelector('a').href,
title: resultElement.querySelector('.title').innerText,
extract: resultElement.querySelector('.extract').innerText,
- curated: resultElement.querySelector('.curate-approve').isValidated()
+ validated: resultElement.querySelector('.curate-approve').classList.contains('validated'),
+ source: resultElement.querySelector('.source').innerText,
}
results.push(result);
}
diff --git a/mwmbl/platform/curate.py b/mwmbl/platform/curate.py
index 782195f..16ffa86 100644
--- a/mwmbl/platform/curate.py
+++ b/mwmbl/platform/curate.py
@@ -1,5 +1,5 @@
from logging import getLogger
-from typing import Any
+from typing import Any, Optional
from urllib.parse import parse_qs
from ninja import Router, NinjaAPI
@@ -8,7 +8,7 @@ from mwmbl.indexer.update_urls import get_datetime_from_timestamp
from mwmbl.models import UserCuration
from mwmbl.platform.data import CurateBegin, CurateMove, CurateDelete, CurateAdd, CurateValidate, \
make_curation_type
-from mwmbl.tinysearchengine.indexer import TinyIndex, Document
+from mwmbl.tinysearchengine.indexer import TinyIndex, Document, DocumentState
from mwmbl.tokenizer import tokenize
from mwmbl.utils import add_term_info, add_term_infos
@@ -67,7 +67,14 @@ def create_router(index_path: str, version: str) -> NinjaAPI:
term = " ".join(tokens)
documents = [
- Document(result.title, result.url, result.extract, MAX_CURATED_SCORE - i, term, result.curated)
+ Document(
+ title=result.title,
+ url=result.url,
+ extract=result.extract,
+ score=MAX_CURATED_SCORE - i,
+ term=term,
+ state=_get_document_state(result.validated, result.source),
+ )
for i, result in enumerate(curation.results)
]
@@ -87,3 +94,12 @@ def create_router(index_path: str, version: str) -> NinjaAPI:
return router
+def _get_document_state(validated: bool, source: str) -> Optional[DocumentState]:
+ if validated:
+ return DocumentState.VALIDATED
+ elif source.lower() == "user":
+ return DocumentState.FROM_USER
+ elif source.lower() == "google":
+ return DocumentState.FROM_GOOGLE
+ else:
+ return DocumentState.CURATED
diff --git a/mwmbl/platform/data.py b/mwmbl/platform/data.py
index 93b8f41..5929194 100644
--- a/mwmbl/platform/data.py
+++ b/mwmbl/platform/data.py
@@ -8,7 +8,8 @@ class Result(Schema):
url: str
title: str
extract: str
- curated: bool
+ validated: bool
+ source: str
class CurateBegin(Schema):
diff --git a/mwmbl/templates/result.html b/mwmbl/templates/result.html
index 339d293..c63b8fb 100644
--- a/mwmbl/templates/result.html
+++ b/mwmbl/templates/result.html
@@ -3,7 +3,7 @@
<div class="result-container">
<div class="result-link">
<a href="{{result.url}}">
- <p><span class='link'>{{result.url}}</span> <span class="via">— found via {{result.source|title}}</span></p>
+ <p><span class='link'>{{result.url}}</span> <span class="via">— found via <span class="source">{{result.source|title}}</span></span></p>
<p class='title'>{{result.title|strengthen}}</p>
</a>
<p class='extract'>{{result.extract|strengthen}}</p>
diff --git a/mwmbl/tinysearchengine/indexer.py b/mwmbl/tinysearchengine/indexer.py
index 33b0d02..156c519 100644
--- a/mwmbl/tinysearchengine/indexer.py
+++ b/mwmbl/tinysearchengine/indexer.py
@@ -31,6 +31,9 @@ def astuple(dc):
class DocumentState(IntEnum):
+ """
+ The state of the document in the index. A value of None indicates a document from our normal crawling procedure.
+ """
CURATED = 0
VALIDATED = 1
FROM_USER = 2
diff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.py
index 9ccec5d..4ca4505 100644
--- a/mwmbl/tinysearchengine/rank.py
+++ b/mwmbl/tinysearchengine/rank.py
@@ -175,7 +175,7 @@ class Ranker:
# Check for curation
curation_term = " ".join(terms)
curation_items = self.tiny_index.retrieve(curation_term)
- curated_items = [d for d in curation_items if d.state in {DocumentState.CURATED, DocumentState.VALIDATED}
+ curated_items = [d for d in curation_items if d.state is not None
and d.term == curation_term]
if len(curated_items) > 0: