Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

4.3 KB
Raw Blame
#!/usr/bin/env python3
"""
Smoke-test for the user_ids / last_crawled feature on beta.

Fetches existing results for a single query term from a local dev server,
re-crawls a small number of those URLs, posts them back to the results
endpoint, then checks that user_ids is populated in the raw search response.

Usage:
    MWMBL_API_KEY=<key> MWMBL_CONTACT_INFO=<[email protected]> \
        uv run python scripts/verify_user_ids_beta.py <term>
"""

import argparse
import json
import os
import sys
import time

import redis
import requests

# Must be set before importing retrieve (used in the User-Agent string)
os.environ.setdefault("MWMBL_CONTACT_INFO", "[email protected]")

from mwmbl.crawler.retrieve import CRAWLER_VERSION, crawl_url
from mwmbl.settings import REDIS_URL

_redis = None


def _get_redis() -> redis.Redis:
    """Get or create Redis connection for robots.txt caching."""
    global _redis
    if _redis is None:
        _redis = redis.from_url(REDIS_URL, decode_responses=True)
    return _redis


BETA_BASE = "http://localhost:8000"
MAX_URLS = 5


def fetch_raw(term: str) -> list[dict]:
    url = f"{BETA_BASE}/api/v1/search/raw?s={requests.utils.quote(term)}"
    resp = requests.get(url, timeout=15)
    resp.raise_for_status()
    return resp.json()["results"]


def main():
    parser = argparse.ArgumentParser(description="Verify user_ids tracking on beta")
    parser.add_argument("term", help="Search term to crawl and verify")
    args = parser.parse_args()

    api_key = os.environ.get("MWMBL_API_KEY", "").strip()
    if not api_key:
        print("ERROR: set MWMBL_API_KEY to a crawl-scoped API key", file=sys.stderr)
        sys.exit(1)

    term = args.term

    print(f"[1/4] Fetching existing raw results for '{term}' from beta...")
    existing = fetch_raw(term)
    print(f"      {len(existing)} results found")

    urls = [r["url"] for r in existing[:MAX_URLS]]
    if not urls:
        print("      No URLs to crawl — try a more common search term.")
        sys.exit(1)

    print(f"\n[2/4] Crawling {len(urls)} URL(s)...")
    results = []
    for url in urls:
        print(f"      {url}")
        raw = crawl_url(url, _get_redis())
        content = raw.get("content")
        if content and not raw.get("error") and content.get("title"):
            last_crawled = int(raw["timestamp"] / 1000)  # ms → seconds
            results.append(
                {
                    "url": raw["url"],
                    "title": content["title"],
                    "extract": content.get("extract", ""),
                    "last_crawled": last_crawled,
                }
            )
            print(f"        OK — {content['title'][:70]}")
        else:
            err = (raw.get("error") or {}).get("name", "unknown error")
            print(f"        skipped ({err})")

    if not results:
        print("\nNo successful crawls — cannot post or verify.")
        sys.exit(1)

    print(f"\n[3/4] Posting {len(results)} result(s) to beta...")
    payload = {
        "results": results,
        "crawler_version": CRAWLER_VERSION,
    }
    resp = requests.post(
        f"{BETA_BASE}/api/v1/crawler/results",
        json=payload,
        headers={"X-API-Key": api_key},
        timeout=30,
    )
    print(f"      {resp.status_code}: {resp.text}")
    resp.raise_for_status()

    print("\n[4/4] Verifying user_ids in raw results (waiting 2s for index)...")
    time.sleep(2)
    updated = fetch_raw(term)
    print("Updated page", json.dumps(updated, indent=2))

    crawled_urls = {r["url"] for r in results}
    passed, failed = [], []
    for item in updated:
        if item["url"] not in crawled_urls:
            continue
        if item.get("user_ids"):
            passed.append(item)
        else:
            failed.append(item)

    for item in passed:
        print(f"      PASS  {item['url']}")
        print(f"            user_ids={item['user_ids']}  last_crawled={item['last_crawled']}")
    for item in failed:
        print(f"      FAIL  {item['url']}  (user_ids not set)")

    if passed and not failed:
        print(f"\nAll {len(passed)} crawled URL(s) have user_ids set. ✓")
    elif passed:
        print(f"\nPartial: {len(passed)} passed, {len(failed)} failed.")
        sys.exit(1)
    else:
        print("\nFAIL: no crawled items have user_ids in the index.")
        sys.exit(1)


if __name__ == "__main__":
    main()