An open source, non-profit web search engine
Include inputs in results so we can use htmx
Daoud Clarke1ed8d0cfc00d
diff --git a/analyse/search.py b/analyse/search.pydeleted file mode 100644index 8f0f1de..0000000--- a/analyse/search.py+++ /dev/null@@ -1,32 +0,0 @@-import logging-import sys-from itertools import islice--from mwmbl.indexer import INDEX_PATH-from mwmbl.tinysearchengine.completer import Completer-from mwmbl.tinysearchengine import TinyIndex, Document-from mwmbl.tinysearchengine.rank import HeuristicRanker--logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)---def clean(sequence):- return ''.join(x['value'] for x in sequence)---def run():- with TinyIndex(Document, INDEX_PATH) as tiny_index:- completer = Completer()- ranker = HeuristicRanker(tiny_index, completer)- items = ranker.search('jasper fforde', [])- print()- if items:- for i, item in enumerate(islice(items, 10)):- print(f"{i + 1}. {item['url']}")- print(clean(item['title']))- print(clean(item['extract']))- print()---if __name__ == '__main__':- run()diff --git a/devdata/index-v2.tinysearch b/devdata/index-v2.tinysearchindex 5a9cf2f..eb18b3a 100644Binary files a/devdata/index-v2.tinysearch and b/devdata/index-v2.tinysearch differdiff --git a/mwmbl/models.py b/mwmbl/models.pyindex 30d6e01..24896b5 100644--- a/mwmbl/models.py+++ b/mwmbl/models.py@@ -7,9 +7,21 @@ class MwmblUser(AbstractUser):class UserCuration(models.Model):+ """+ Deprecated - use Curation instead+ """user = models.ForeignKey(MwmblUser, on_delete=models.CASCADE)timestamp = models.DateTimeField()url = models.CharField(max_length=300)results = models.JSONField()curation_type = models.CharField(max_length=20)curation = models.JSONField()+++class Curation(models.Model):+ user = models.ForeignKey(MwmblUser, on_delete=models.CASCADE)+ timestamp = models.DateTimeField()+ url = models.CharField(max_length=300)+ original_results = models.JSONField()+ new_results = models.JSONField()+ num_changes = models.IntegerField(default=0)diff --git a/mwmbl/templates/result.html b/mwmbl/templates/result.htmlindex c63b8fb..7e573fc 100644--- a/mwmbl/templates/result.html+++ b/mwmbl/templates/result.html@@ -3,15 +3,22 @@<div class="result-container"><div class="result-link"><a href="{{result.url}}">- <p><span class='link'>{{result.url}}</span> <span class="via">— found via <span class="source">{{result.source|title}}</span></span></p>- <p class='title'>{{result.title|strengthen}}</p>+ <p><span class='link'>{{result.url}}</span> <span class="via">— found via <span class="source">{{result.state|convert_state_to_source|title}}</span></span></p>+ <p class='title'>{{result.title|format_for_query:query}}</p></a>- <p class='extract'>{{result.extract|strengthen}}</p>+ <p class='extract'>{{result.extract|format_for_query:query}}</p>+ <input type="hidden" name="url" value="{{result.url}}" />+ <input type="hidden" name="title" value="{{result.title}}" />+ <input type="hidden" name="extract" value="{{result.extract}}" /></div><div class="curation-buttons">- <span class="button handle">↕ Move</span>- <button class="button curate-delete" is="mwmbl-delete-button">✕ Delete</button>- <button class="button curate-approve" is="mwmbl-validate-button">✓ Looks good</button>+ {% if result.state.name == 'VALIDATED' %}+ <button class="button move-up">↑</button>+ <button class="button move-down">↓</button>+ <button class="button curate-approve" is="mwmbl-validate-button">✓ Approved</button>+ {% else %}+ <button class="button curate-approve" is="mwmbl-validate-button">✓ Approve</button>+ {% endif %}</div></div></li>diff --git a/mwmbl/templatetags/result_filters.py b/mwmbl/templatetags/result_filters.pyindex 8acf02d..62aeba9 100644--- a/mwmbl/templatetags/result_filters.py+++ b/mwmbl/templatetags/result_filters.py@@ -1,18 +1,32 @@+import re+from django.template import Libraryfrom django.utils.html import conditional_escapefrom django.utils.safestring import mark_safe+from mwmbl.format import get_query_regex, DOCUMENT_SOURCES+from mwmbl.tokenizer import tokenize+register = Library()@register.filter(needs_autoescape=True)-def strengthen(spans, autoescape=True):+def format_for_query(text: str, query: str, autoescape=True):+ escape = conditional_escape if autoescape else lambda x: x+ tokens = tokenize(query)+ pattern = get_query_regex(tokens, True, False)+ matches = re.finditer(pattern, text, re.IGNORECASE)+ formatted = []+ start = 0+ for match in matches:+ formatted.append(escape(text[start:match.start()]))+ formatted.append(f"<strong>{escape(text[match.start():match.end()])}</strong>")+ start = match.end()+ formatted.append(escape(text[start:]))+ return mark_safe("".join(formatted))++[email protected](needs_autoescape=True)+def convert_state_to_source(state: str, autoescape=True):escape = conditional_escape if autoescape else lambda x: x- strengthened = []- for span in spans:- escaped_value = escape(span["value"])- if span["is_bold"]:- strengthened.append(f"<strong>{escaped_value}</strong>")- else:- strengthened.append(escaped_value)- return mark_safe("".join(strengthened))+ return escape(DOCUMENT_SOURCES[state] if state else 'mwmbl')diff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.pyindex c58d792..ac4a657 100644--- a/mwmbl/tinysearchengine/rank.py+++ b/mwmbl/tinysearchengine/rank.py@@ -128,22 +128,21 @@ class Ranker:def order_results(self, terms, pages, is_complete):pass- def search(self, s: str, additional_results: list[Document]):+ def search(self, s: str, additional_results: list[Document]) -> list[Document]:results, terms, _ = self.get_results(s, additional_results)is_complete = s.endswith(' ')pattern = get_query_regex(terms, is_complete, False)- formatted_results = []+ ranked_results = []seen_urls = set()for result in results:if result.url in seen_urls:continue- formatted_result = format_result_with_pattern(pattern, result)- formatted_results.append(formatted_result)+ ranked_results.append(result)seen_urls.add(result.url)- logger.info("Return results: %d", len(formatted_results))- return formatted_results+ logger.info("Return results: %d", len(ranked_results))+ return ranked_resultsdef complete(self, q: str):ordered_results, terms, completions = self.get_results(q, [])diff --git a/mwmbl/tinysearchengine/search.py b/mwmbl/tinysearchengine/search.pyindex 416527d..021d2cd 100644--- a/mwmbl/tinysearchengine/search.py+++ b/mwmbl/tinysearchengine/search.py@@ -2,6 +2,7 @@ from logging import getLoggerfrom ninja import NinjaAPI+from mwmbl.format import format_resultfrom mwmbl.tinysearchengine.rank import HeuristicRankerlogger = getLogger(__name__)@@ -15,7 +16,8 @@ def create_router(ranker: HeuristicRanker, version: str) -> NinjaAPI:@router.get("")def search(request, s: str):- return ranker.search(s, [])+ results = ranker.search(s, [])+ return [format_result(result, s) for result in results]@router.get("/complete")def complete(request, q: str):diff --git a/mwmbl/views.py b/mwmbl/views.pyindex e33e841..ea7cd9d 100644--- a/mwmbl/views.py+++ b/mwmbl/views.py@@ -139,5 +139,5 @@ def fetch_url(request):result = Document(title=title, url=url, extract=extract, score=0.0, state=DocumentState.FROM_USER)return render(request, "result.html", {- "result": format_result(result, query),+ "result": result, "query": query})