An open source, non-profit web search engine
Improve admin for creating API key
Daoud Clarkea12b278993a1
diff --git a/mwmbl/admin.py b/mwmbl/admin.pyindex a015962..42b0dc5 100644--- a/mwmbl/admin.py+++ b/mwmbl/admin.py@@ -1,12 +1,53 @@+from django import formsfrom django.contrib import adminfrom django.contrib.admin import ModelAdminfrom django.contrib.auth.admin import UserAdmin-from mwmbl.models import MwmblUser, OldIndex, Curation, FlagCuration, DomainSubmission, ApiKey+from mwmbl.models import MwmblUser, OldIndex, Curation, FlagCuration, DomainSubmission, ApiKey, generate_api_key+++class ApiKeyForm(forms.ModelForm):+ scopes = forms.MultipleChoiceField(+ choices=ApiKey.Scope.choices,+ widget=forms.CheckboxSelectMultiple,+ required=False,+ )++ class Meta:+ model = ApiKey+ fields = "__all__"+++class ApiKeyAdmin(ModelAdmin):+ form = ApiKeyForm+ list_display = ("user", "name", "scopes", "created_on")+ readonly_fields = ("key", "created_on")+ fields = ("user", "name", "scopes", "key", "created_on")+ _pending_raw_key = None++ def save_model(self, request, obj, form, change):+ if not change:+ raw_key, key_hash = generate_api_key()+ obj.key = key_hash+ self._pending_raw_key = raw_key+ super().save_model(request, obj, form, change)+ if self._pending_raw_key:+ self.message_user(+ request,+ f"API key created. Save this key now — it will not be shown again: {self._pending_raw_key}",+ level="warning",+ )+ self._pending_raw_key = None++ def get_readonly_fields(self, request, obj=None):+ if obj:+ return self.readonly_fields + ("user",)+ return self.readonly_fields+admin.site.register(MwmblUser, UserAdmin)admin.site.register(Curation, ModelAdmin)admin.site.register(OldIndex, ModelAdmin)admin.site.register(FlagCuration, ModelAdmin)admin.site.register(DomainSubmission, ModelAdmin)-admin.site.register(ApiKey, ModelAdmin)+admin.site.register(ApiKey, ApiKeyAdmin)diff --git a/scripts/verify_user_ids_beta.py b/scripts/verify_user_ids_beta.pynew file mode 100644index 0000000..d5962e6--- /dev/null+++ b/scripts/verify_user_ids_beta.py@@ -0,0 +1,128 @@+#!/usr/bin/env python3+"""+Smoke-test for the user_ids / last_crawled feature on beta.++Fetches existing results for a single query term from a local dev server,+re-crawls a small number of those URLs, posts them back to the results+endpoint, then checks that user_ids is populated in the raw search response.++Usage:+ MWMBL_API_KEY=<key> MWMBL_CONTACT_INFO=<[email protected]> \+ uv run python scripts/verify_user_ids_beta.py <term>+"""++import argparse+import sys+import os+import time+import json++import requests++# Must be set before importing retrieve (used in the User-Agent string)+os.environ.setdefault("MWMBL_CONTACT_INFO", "[email protected]")++from mwmbl.crawler.retrieve import crawl_url, CRAWLER_VERSION++BETA_BASE = "http://localhost:8000"+MAX_URLS = 5+++def fetch_raw(term: str) -> list[dict]:+ url = f"{BETA_BASE}/api/v1/search/raw?s={requests.utils.quote(term)}"+ resp = requests.get(url, timeout=15)+ resp.raise_for_status()+ return resp.json()["results"]+++def main():+ parser = argparse.ArgumentParser(description="Verify user_ids tracking on beta")+ parser.add_argument("term", help="Search term to crawl and verify")+ args = parser.parse_args()++ api_key = os.environ.get("MWMBL_API_KEY", "").strip()+ if not api_key:+ print("ERROR: set MWMBL_API_KEY to a crawl-scoped API key", file=sys.stderr)+ sys.exit(1)++ term = args.term++ print(f"[1/4] Fetching existing raw results for '{term}' from beta...")+ existing = fetch_raw(term)+ print(f" {len(existing)} results found")++ urls = [r["url"] for r in existing[:MAX_URLS]]+ if not urls:+ print(" No URLs to crawl — try a more common search term.")+ sys.exit(1)++ print(f"\n[2/4] Crawling {len(urls)} URL(s)...")+ results = []+ for url in urls:+ print(f" {url}")+ raw = crawl_url(url)+ content = raw.get("content")+ if content and not raw.get("error") and content.get("title"):+ last_crawled = int(raw["timestamp"] / 1000) # ms → seconds+ results.append({+ "url": raw["url"],+ "title": content["title"],+ "extract": content.get("extract", ""),+ "last_crawled": last_crawled,+ })+ print(f" OK — {content['title'][:70]}")+ else:+ err = (raw.get("error") or {}).get("name", "unknown error")+ print(f" skipped ({err})")++ if not results:+ print("\nNo successful crawls — cannot post or verify.")+ sys.exit(1)++ print(f"\n[3/4] Posting {len(results)} result(s) to beta...")+ payload = {+ "results": results,+ "crawler_version": CRAWLER_VERSION,+ }+ resp = requests.post(+ f"{BETA_BASE}/api/v1/crawler/results",+ json=payload,+ headers={"X-API-Key": api_key},+ timeout=30,+ )+ print(f" {resp.status_code}: {resp.text}")+ resp.raise_for_status()++ print("\n[4/4] Verifying user_ids in raw results (waiting 2s for index)...")+ time.sleep(2)+ updated = fetch_raw(term)+ print("Updated page", json.dumps(updated, indent=2))++ crawled_urls = {r["url"] for r in results}+ passed, failed = [], []+ for item in updated:+ if item["url"] not in crawled_urls:+ continue+ if item.get("user_ids"):+ passed.append(item)+ else:+ failed.append(item)++ for item in passed:+ print(f" PASS {item['url']}")+ print(f" user_ids={item['user_ids']} last_crawled={item['last_crawled']}")+ for item in failed:+ print(f" FAIL {item['url']} (user_ids not set)")++ if passed and not failed:+ print(f"\nAll {len(passed)} crawled URL(s) have user_ids set. ✓")+ elif passed:+ print(f"\nPartial: {len(passed)} passed, {len(failed)} failed.")+ sys.exit(1)+ else:+ print("\nFAIL: no crawled items have user_ids in the index.")+ sys.exit(1)+++if __name__ == "__main__":+ main()