Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Add migrations; remove unused code

Daoud Clarked7dbb9fda673
diff --git a/mwmbl/migrations/0002_curation.py b/mwmbl/migrations/0002_curation.py
new file mode 100644
index 0000000..17a81fb
--- /dev/null
+++ b/mwmbl/migrations/0002_curation.py
@@ -0,0 +1,27 @@
+# Generated by Django 4.2.6 on 2023-12-02 12:42
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('mwmbl', '0001_initial'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='Curation',
+ fields=[
+ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('timestamp', models.DateTimeField()),
+ ('query', models.CharField(max_length=300)),
+ ('original_results', models.JSONField()),
+ ('new_results', models.JSONField()),
+ ('num_changes', models.IntegerField(default=0)),
+ ('user', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+ ],
+ ),
+ ]
diff --git a/mwmbl/migrations/0003_auto_20231203_0614.py b/mwmbl/migrations/0003_auto_20231203_0614.py
new file mode 100644
index 0000000..7a4be6d
--- /dev/null
+++ b/mwmbl/migrations/0003_auto_20231203_0614.py
@@ -0,0 +1,55 @@
+# Generated by Django 4.2.6 on 2023-12-03 06:14
+from itertools import groupby
+from urllib.parse import urlparse, parse_qs
+
+from django.db import migrations
+
+
+def create_curations_from_user_curation(apps, schema_editor):
+ Curation = apps.get_model('mwmbl', 'Curation')
+ UserCuration = apps.get_model('mwmbl', 'UserCuration')
+
+ # Order curations by timestamp
+ curations = UserCuration.objects.all().order_by('timestamp')
+
+ # Group by user and url
+ for (user, url), curations in groupby(curations, lambda c: (c.user, c.url)):
+ curations = list(curations)
+
+ # Get the original results from the first curation
+ original_results = curations[0].results
+
+ # Get the new results from the last curation
+ new_results = curations[-1].results
+
+ # Get the number of changes from the number of curations
+ num_changes = len(curations)
+
+ # Get the timestamp from the first curation
+ timestamp = curations[0].timestamp
+
+ # Get the query from the url
+ parsed_url_query = parse_qs(urlparse(url).query)
+ query = parsed_url_query.get("q", [""])[0]
+
+ # Create the new curation
+ curation = Curation(
+ timestamp=timestamp,
+ query=query,
+ original_results=original_results,
+ new_results=new_results,
+ num_changes=num_changes,
+ user=user,
+ )
+ curation.save()
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('mwmbl', '0002_curation'),
+ ]
+
+ operations = [
+ migrations.RunPython(create_curations_from_user_curation),
+ ]
diff --git a/mwmbl/platform/curate.py b/mwmbl/platform/curate.py
deleted file mode 100644
index 95049d8..0000000
--- a/mwmbl/platform/curate.py
+++ /dev/null
@@ -1,110 +0,0 @@
-from logging import getLogger
-from typing import Any, Optional
-from urllib.parse import parse_qs
-
-from ninja import Router, NinjaAPI
-
-from mwmbl.indexer.update_urls import get_datetime_from_timestamp
-from mwmbl.models import UserCuration
-from mwmbl.platform.data import CurateBegin, CurateMove, CurateDelete, CurateAdd, CurateValidate, \
- make_curation_type
-from mwmbl.tinysearchengine.indexer import TinyIndex, Document, DocumentState
-from mwmbl.tokenizer import tokenize
-from mwmbl.utils import add_term_info, add_term_infos
-
-RESULT_URL = "https://mwmbl.org/?q="
-MAX_CURATED_SCORE = 1_111_111.0
-
-
-logger = getLogger(__name__)
-
-
-def create_router(index_path: str, version: str) -> NinjaAPI:
- router = NinjaAPI(urls_namespace=f"curate-{version}", csrf=True)
-
- @router.post("/begin")
- def user_begin_curate(request, curate_begin: make_curation_type(CurateBegin)):
- return _curate(request, "curate_begin", curate_begin)
-
- @router.post("/move")
- def user_move_result(request, curate_move: make_curation_type(CurateMove)):
- return _curate(request, "curate_move", curate_move)
-
- @router.post("/delete")
- def user_delete_result(request, curate_delete: make_curation_type(CurateDelete)):
- return _curate(request, "curate_delete", curate_delete)
-
- @router.post("/add")
- def user_add_result(request, curate_add: make_curation_type(CurateAdd)):
- return _curate(request, "curate_add", curate_add)
-
- @router.post("/validate")
- def user_add_result(request, curate_validate: make_curation_type(CurateValidate)):
- return _curate(request, "curate_validate", curate_validate)
-
- def _curate(request, curation_type: str, curation: Any):
- user_curation = UserCuration(
- user=request.user,
- timestamp=get_datetime_from_timestamp(curation.timestamp / 1000.0),
- url=curation.url,
- results=curation.dict()["results"],
- curation_type=curation_type,
- curation=curation.curation.dict(),
- )
- user_curation.save()
-
- with TinyIndex(Document, index_path, 'w') as indexer:
- query_string = parse_qs(curation.url)
- if len(query_string) > 1:
- raise ValueError(f"Should be one query string in the URL: {curation.url}")
-
- queries = next(iter(query_string.values()))
- if len(queries) > 1:
- raise ValueError(f"Should be one query value in the URL: {curation.url}")
-
- query = queries[0]
- tokens = tokenize(query)
- term = " ".join(tokens)
-
- documents = [
- Document(
- title=result.title,
- url=result.url,
- extract=result.extract,
- score=MAX_CURATED_SCORE - i,
- term=term,
- state=_get_document_state(result.validated, result.source),
- )
- for i, result in enumerate(curation.results)
- ]
-
- page_index = indexer.get_key_page_index(term)
- existing_documents_no_terms = indexer.get_page(page_index)
- existing_documents = add_term_infos(existing_documents_no_terms, indexer, page_index)
- other_documents = [doc for doc in existing_documents if doc.term != term]
- logger.info(f"Found {len(other_documents)} other documents for term {term} at page {page_index} "
- f"with terms { {doc.term for doc in other_documents} }")
-
- all_documents = documents + other_documents
- logger.info(f"Storing {len(all_documents)} documents at page {page_index}")
- indexer.store_in_page(page_index, all_documents)
-
- return {"curation": "ok"}
-
- return router
-
-
-def _get_document_state(validated: bool, source: str) -> Optional[DocumentState]:
- if validated:
- if source.lower() == "user":
- return DocumentState.FROM_USER_APPROVED
- elif source.lower() == "google":
- return DocumentState.FROM_GOOGLE_APPROVED
- else:
- return DocumentState.ORGANIC_APPROVED
- elif source.lower() == "user":
- return DocumentState.FROM_USER
- elif source.lower() == "google":
- return DocumentState.FROM_GOOGLE
- else:
- return None
diff --git a/mwmbl/urls.py b/mwmbl/urls.py
index bd7a764..0f62bc3 100644
--- a/mwmbl/urls.py
+++ b/mwmbl/urls.py
@@ -18,8 +18,7 @@ from django.contrib import admin
from django.urls import path, include
import mwmbl.crawler.app as crawler
-from mwmbl.platform import curate
-from mwmbl.search_setup import queued_batches, index_path, ranker, batch_cache
+from mwmbl.search_setup import queued_batches, ranker, batch_cache
from mwmbl.tinysearchengine import search
from mwmbl.views import home_fragment, fetch_url, index, approve
@@ -35,10 +34,8 @@ urlpatterns = [
# TODO: this is the old API, deprecated and to be removed once all clients have moved over
path("search/", search.create_router(ranker, "0.1").urls),
path("crawler/", crawler.create_router(batch_cache=batch_cache, queued_batches=queued_batches, version="0.1").urls),
- path("curation/", curate.create_router(index_path, version="0.1").urls),
# New API
path("api/v1/search/", search.create_router(ranker, "1.0.0").urls),
path("api/v1/crawler/", crawler.create_router(batch_cache=batch_cache, queued_batches=queued_batches, version="1.0.0").urls),
- path("api/v1/curation/", curate.create_router(index_path, version="1.0.0").urls),
]