Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Improve admin for creating API key

Daoud Clarkea12b278993a1
diff --git a/mwmbl/admin.py b/mwmbl/admin.py
index a015962..42b0dc5 100644
--- a/mwmbl/admin.py
+++ b/mwmbl/admin.py
@@ -1,12 +1,53 @@
+from django import forms
from django.contrib import admin
from django.contrib.admin import ModelAdmin
from django.contrib.auth.admin import UserAdmin
-from mwmbl.models import MwmblUser, OldIndex, Curation, FlagCuration, DomainSubmission, ApiKey
+from mwmbl.models import MwmblUser, OldIndex, Curation, FlagCuration, DomainSubmission, ApiKey, generate_api_key
+
+
+class ApiKeyForm(forms.ModelForm):
+ scopes = forms.MultipleChoiceField(
+ choices=ApiKey.Scope.choices,
+ widget=forms.CheckboxSelectMultiple,
+ required=False,
+ )
+
+ class Meta:
+ model = ApiKey
+ fields = "__all__"
+
+
+class ApiKeyAdmin(ModelAdmin):
+ form = ApiKeyForm
+ list_display = ("user", "name", "scopes", "created_on")
+ readonly_fields = ("key", "created_on")
+ fields = ("user", "name", "scopes", "key", "created_on")
+ _pending_raw_key = None
+
+ def save_model(self, request, obj, form, change):
+ if not change:
+ raw_key, key_hash = generate_api_key()
+ obj.key = key_hash
+ self._pending_raw_key = raw_key
+ super().save_model(request, obj, form, change)
+ if self._pending_raw_key:
+ self.message_user(
+ request,
+ f"API key created. Save this key now — it will not be shown again: {self._pending_raw_key}",
+ level="warning",
+ )
+ self._pending_raw_key = None
+
+ def get_readonly_fields(self, request, obj=None):
+ if obj:
+ return self.readonly_fields + ("user",)
+ return self.readonly_fields
+
admin.site.register(MwmblUser, UserAdmin)
admin.site.register(Curation, ModelAdmin)
admin.site.register(OldIndex, ModelAdmin)
admin.site.register(FlagCuration, ModelAdmin)
admin.site.register(DomainSubmission, ModelAdmin)
-admin.site.register(ApiKey, ModelAdmin)
+admin.site.register(ApiKey, ApiKeyAdmin)
diff --git a/scripts/verify_user_ids_beta.py b/scripts/verify_user_ids_beta.py
new file mode 100644
index 0000000..d5962e6
--- /dev/null
+++ b/scripts/verify_user_ids_beta.py
@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+"""
+Smoke-test for the user_ids / last_crawled feature on beta.
+
+Fetches existing results for a single query term from a local dev server,
+re-crawls a small number of those URLs, posts them back to the results
+endpoint, then checks that user_ids is populated in the raw search response.
+
+Usage:
+ MWMBL_API_KEY=<key> MWMBL_CONTACT_INFO=<[email protected]> \
+ uv run python scripts/verify_user_ids_beta.py <term>
+"""
+
+import argparse
+import sys
+import os
+import time
+import json
+
+import requests
+
+# Must be set before importing retrieve (used in the User-Agent string)
+os.environ.setdefault("MWMBL_CONTACT_INFO", "[email protected]")
+
+from mwmbl.crawler.retrieve import crawl_url, CRAWLER_VERSION
+
+BETA_BASE = "http://localhost:8000"
+MAX_URLS = 5
+
+
+def fetch_raw(term: str) -> list[dict]:
+ url = f"{BETA_BASE}/api/v1/search/raw?s={requests.utils.quote(term)}"
+ resp = requests.get(url, timeout=15)
+ resp.raise_for_status()
+ return resp.json()["results"]
+
+
+def main():
+ parser = argparse.ArgumentParser(description="Verify user_ids tracking on beta")
+ parser.add_argument("term", help="Search term to crawl and verify")
+ args = parser.parse_args()
+
+ api_key = os.environ.get("MWMBL_API_KEY", "").strip()
+ if not api_key:
+ print("ERROR: set MWMBL_API_KEY to a crawl-scoped API key", file=sys.stderr)
+ sys.exit(1)
+
+ term = args.term
+
+ print(f"[1/4] Fetching existing raw results for '{term}' from beta...")
+ existing = fetch_raw(term)
+ print(f" {len(existing)} results found")
+
+ urls = [r["url"] for r in existing[:MAX_URLS]]
+ if not urls:
+ print(" No URLs to crawl — try a more common search term.")
+ sys.exit(1)
+
+ print(f"\n[2/4] Crawling {len(urls)} URL(s)...")
+ results = []
+ for url in urls:
+ print(f" {url}")
+ raw = crawl_url(url)
+ content = raw.get("content")
+ if content and not raw.get("error") and content.get("title"):
+ last_crawled = int(raw["timestamp"] / 1000) # ms → seconds
+ results.append({
+ "url": raw["url"],
+ "title": content["title"],
+ "extract": content.get("extract", ""),
+ "last_crawled": last_crawled,
+ })
+ print(f" OK — {content['title'][:70]}")
+ else:
+ err = (raw.get("error") or {}).get("name", "unknown error")
+ print(f" skipped ({err})")
+
+ if not results:
+ print("\nNo successful crawls — cannot post or verify.")
+ sys.exit(1)
+
+ print(f"\n[3/4] Posting {len(results)} result(s) to beta...")
+ payload = {
+ "results": results,
+ "crawler_version": CRAWLER_VERSION,
+ }
+ resp = requests.post(
+ f"{BETA_BASE}/api/v1/crawler/results",
+ json=payload,
+ headers={"X-API-Key": api_key},
+ timeout=30,
+ )
+ print(f" {resp.status_code}: {resp.text}")
+ resp.raise_for_status()
+
+ print("\n[4/4] Verifying user_ids in raw results (waiting 2s for index)...")
+ time.sleep(2)
+ updated = fetch_raw(term)
+ print("Updated page", json.dumps(updated, indent=2))
+
+ crawled_urls = {r["url"] for r in results}
+ passed, failed = [], []
+ for item in updated:
+ if item["url"] not in crawled_urls:
+ continue
+ if item.get("user_ids"):
+ passed.append(item)
+ else:
+ failed.append(item)
+
+ for item in passed:
+ print(f" PASS {item['url']}")
+ print(f" user_ids={item['user_ids']} last_crawled={item['last_crawled']}")
+ for item in failed:
+ print(f" FAIL {item['url']} (user_ids not set)")
+
+ if passed and not failed:
+ print(f"\nAll {len(passed)} crawled URL(s) have user_ids set. ✓")
+ elif passed:
+ print(f"\nPartial: {len(passed)} passed, {len(failed)} failed.")
+ sys.exit(1)
+ else:
+ print("\nFAIL: no crawled items have user_ids in the index.")
+ sys.exit(1)
+
+
+if __name__ == "__main__":
+ main()