Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Include inputs in results so we can use htmx

Daoud Clarke1ed8d0cfc00d
diff --git a/analyse/search.py b/analyse/search.py
deleted file mode 100644
index 8f0f1de..0000000
--- a/analyse/search.py
+++ /dev/null
@@ -1,32 +0,0 @@
-import logging
-import sys
-from itertools import islice
-
-from mwmbl.indexer import INDEX_PATH
-from mwmbl.tinysearchengine.completer import Completer
-from mwmbl.tinysearchengine import TinyIndex, Document
-from mwmbl.tinysearchengine.rank import HeuristicRanker
-
-logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
-
-
-def clean(sequence):
- return ''.join(x['value'] for x in sequence)
-
-
-def run():
- with TinyIndex(Document, INDEX_PATH) as tiny_index:
- completer = Completer()
- ranker = HeuristicRanker(tiny_index, completer)
- items = ranker.search('jasper fforde', [])
- print()
- if items:
- for i, item in enumerate(islice(items, 10)):
- print(f"{i + 1}. {item['url']}")
- print(clean(item['title']))
- print(clean(item['extract']))
- print()
-
-
-if __name__ == '__main__':
- run()
diff --git a/devdata/index-v2.tinysearch b/devdata/index-v2.tinysearch
index 5a9cf2f..eb18b3a 100644
Binary files a/devdata/index-v2.tinysearch and b/devdata/index-v2.tinysearch differ
diff --git a/mwmbl/models.py b/mwmbl/models.py
index 30d6e01..24896b5 100644
--- a/mwmbl/models.py
+++ b/mwmbl/models.py
@@ -7,9 +7,21 @@ class MwmblUser(AbstractUser):
class UserCuration(models.Model):
+ """
+ Deprecated - use Curation instead
+ """
user = models.ForeignKey(MwmblUser, on_delete=models.CASCADE)
timestamp = models.DateTimeField()
url = models.CharField(max_length=300)
results = models.JSONField()
curation_type = models.CharField(max_length=20)
curation = models.JSONField()
+
+
+class Curation(models.Model):
+ user = models.ForeignKey(MwmblUser, on_delete=models.CASCADE)
+ timestamp = models.DateTimeField()
+ url = models.CharField(max_length=300)
+ original_results = models.JSONField()
+ new_results = models.JSONField()
+ num_changes = models.IntegerField(default=0)
diff --git a/mwmbl/templates/result.html b/mwmbl/templates/result.html
index c63b8fb..7e573fc 100644
--- a/mwmbl/templates/result.html
+++ b/mwmbl/templates/result.html
@@ -3,15 +3,22 @@
<div class="result-container">
<div class="result-link">
<a href="{{result.url}}">
- <p><span class='link'>{{result.url}}</span> <span class="via">— found via <span class="source">{{result.source|title}}</span></span></p>
- <p class='title'>{{result.title|strengthen}}</p>
+ <p><span class='link'>{{result.url}}</span> <span class="via">— found via <span class="source">{{result.state|convert_state_to_source|title}}</span></span></p>
+ <p class='title'>{{result.title|format_for_query:query}}</p>
</a>
- <p class='extract'>{{result.extract|strengthen}}</p>
+ <p class='extract'>{{result.extract|format_for_query:query}}</p>
+ <input type="hidden" name="url" value="{{result.url}}" />
+ <input type="hidden" name="title" value="{{result.title}}" />
+ <input type="hidden" name="extract" value="{{result.extract}}" />
</div>
<div class="curation-buttons">
- <span class="button handle">↕ Move</span>
- <button class="button curate-delete" is="mwmbl-delete-button">✕ Delete</button>
- <button class="button curate-approve" is="mwmbl-validate-button">✓ Looks good</button>
+ {% if result.state.name == 'VALIDATED' %}
+ <button class="button move-up">↑</button>
+ <button class="button move-down">↓</button>
+ <button class="button curate-approve" is="mwmbl-validate-button">✓ Approved</button>
+ {% else %}
+ <button class="button curate-approve" is="mwmbl-validate-button">✓ Approve</button>
+ {% endif %}
</div>
</div>
</li>
diff --git a/mwmbl/templatetags/result_filters.py b/mwmbl/templatetags/result_filters.py
index 8acf02d..62aeba9 100644
--- a/mwmbl/templatetags/result_filters.py
+++ b/mwmbl/templatetags/result_filters.py
@@ -1,18 +1,32 @@
+import re
+
from django.template import Library
from django.utils.html import conditional_escape
from django.utils.safestring import mark_safe
+from mwmbl.format import get_query_regex, DOCUMENT_SOURCES
+from mwmbl.tokenizer import tokenize
+
register = Library()
@register.filter(needs_autoescape=True)
-def strengthen(spans, autoescape=True):
+def format_for_query(text: str, query: str, autoescape=True):
+ escape = conditional_escape if autoescape else lambda x: x
+ tokens = tokenize(query)
+ pattern = get_query_regex(tokens, True, False)
+ matches = re.finditer(pattern, text, re.IGNORECASE)
+ formatted = []
+ start = 0
+ for match in matches:
+ formatted.append(escape(text[start:match.start()]))
+ formatted.append(f"<strong>{escape(text[match.start():match.end()])}</strong>")
+ start = match.end()
+ formatted.append(escape(text[start:]))
+ return mark_safe("".join(formatted))
+
+
[email protected](needs_autoescape=True)
+def convert_state_to_source(state: str, autoescape=True):
escape = conditional_escape if autoescape else lambda x: x
- strengthened = []
- for span in spans:
- escaped_value = escape(span["value"])
- if span["is_bold"]:
- strengthened.append(f"<strong>{escaped_value}</strong>")
- else:
- strengthened.append(escaped_value)
- return mark_safe("".join(strengthened))
+ return escape(DOCUMENT_SOURCES[state] if state else 'mwmbl')
diff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.py
index c58d792..ac4a657 100644
--- a/mwmbl/tinysearchengine/rank.py
+++ b/mwmbl/tinysearchengine/rank.py
@@ -128,22 +128,21 @@ class Ranker:
def order_results(self, terms, pages, is_complete):
pass
- def search(self, s: str, additional_results: list[Document]):
+ def search(self, s: str, additional_results: list[Document]) -> list[Document]:
results, terms, _ = self.get_results(s, additional_results)
is_complete = s.endswith(' ')
pattern = get_query_regex(terms, is_complete, False)
- formatted_results = []
+ ranked_results = []
seen_urls = set()
for result in results:
if result.url in seen_urls:
continue
- formatted_result = format_result_with_pattern(pattern, result)
- formatted_results.append(formatted_result)
+ ranked_results.append(result)
seen_urls.add(result.url)
- logger.info("Return results: %d", len(formatted_results))
- return formatted_results
+ logger.info("Return results: %d", len(ranked_results))
+ return ranked_results
def complete(self, q: str):
ordered_results, terms, completions = self.get_results(q, [])
diff --git a/mwmbl/tinysearchengine/search.py b/mwmbl/tinysearchengine/search.py
index 416527d..021d2cd 100644
--- a/mwmbl/tinysearchengine/search.py
+++ b/mwmbl/tinysearchengine/search.py
@@ -2,6 +2,7 @@ from logging import getLogger
from ninja import NinjaAPI
+from mwmbl.format import format_result
from mwmbl.tinysearchengine.rank import HeuristicRanker
logger = getLogger(__name__)
@@ -15,7 +16,8 @@ def create_router(ranker: HeuristicRanker, version: str) -> NinjaAPI:
@router.get("")
def search(request, s: str):
- return ranker.search(s, [])
+ results = ranker.search(s, [])
+ return [format_result(result, s) for result in results]
@router.get("/complete")
def complete(request, q: str):
diff --git a/mwmbl/views.py b/mwmbl/views.py
index e33e841..ea7cd9d 100644
--- a/mwmbl/views.py
+++ b/mwmbl/views.py
@@ -139,5 +139,5 @@ def fetch_url(request):
result = Document(title=title, url=url, extract=extract, score=0.0, state=DocumentState.FROM_USER)
return render(request, "result.html", {
- "result": format_result(result, query),
+ "result": result, "query": query
})