Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Add examples to search API

Daoud Clarked9cf2748a33b
diff --git a/mwmbl/tinysearchengine/search.py b/mwmbl/tinysearchengine/search.py
index 269830a..1086c0f 100644
--- a/mwmbl/tinysearchengine/search.py
+++ b/mwmbl/tinysearchengine/search.py
@@ -1,7 +1,8 @@
import dataclasses
from logging import getLogger
+from typing import Optional
-from ninja import NinjaAPI, Router
+from ninja import NinjaAPI, Router, Schema
from mwmbl.format import format_result
from mwmbl.tinysearchengine.indexer import Document
@@ -16,33 +17,178 @@ SCORE_THRESHOLD = 0.25
router = Router(tags=["Search"])
+# ---------------------------------------------------------------------------
+# Response schemas
+# ---------------------------------------------------------------------------
+
+class TextSegment(Schema):
+ """A fragment of a title or extract, optionally highlighted."""
+ value: str
+ is_bold: bool
+
+ class Config:
+ json_schema_extra = {
+ "examples": [
+ {"value": "The ", "is_bold": False},
+ {"value": "Python", "is_bold": True},
+ {"value": " Tutorial", "is_bold": False},
+ ]
+ }
+
+
+class SearchResult(Schema):
+ """A single formatted search result returned by the main search endpoint."""
+ url: str
+ title: list[TextSegment]
+ extract: list[TextSegment]
+ source: str
+
+ class Config:
+ json_schema_extra = {
+ "examples": [
+ {
+ "url": "https://docs.python.org/3/tutorial/",
+ "title": [
+ {"value": "The ", "is_bold": False},
+ {"value": "Python", "is_bold": True},
+ {"value": " Tutorial — Python 3 documentation", "is_bold": False},
+ ],
+ "extract": [
+ {"value": "Python", "is_bold": True},
+ {"value": " is an easy to learn, powerful programming language.", "is_bold": False},
+ ],
+ "source": "mwmbl",
+ }
+ ]
+ }
+
+
+class RawDocument(Schema):
+ """A raw, unformatted document as stored in the index."""
+ title: str
+ url: str
+ extract: str
+ score: Optional[float] = None
+ term: Optional[str] = None
+ state: Optional[int] = None
+
+ class Config:
+ json_schema_extra = {
+ "examples": [
+ {
+ "title": "The Python Tutorial",
+ "url": "https://docs.python.org/3/tutorial/",
+ "extract": "Python is an easy to learn, powerful programming language.",
+ "score": 0.92,
+ "term": "python tutorial",
+ "state": None,
+ }
+ ]
+ }
+
+
+class RawSearchResponse(Schema):
+ """Response from the raw search endpoint."""
+ query: str
+ results: list[RawDocument]
+
+ class Config:
+ json_schema_extra = {
+ "examples": [
+ {
+ "query": "python tutorial",
+ "results": [
+ {
+ "title": "The Python Tutorial",
+ "url": "https://docs.python.org/3/tutorial/",
+ "extract": "Python is an easy to learn, powerful programming language.",
+ "score": 0.92,
+ "term": "python tutorial",
+ "state": None,
+ }
+ ],
+ }
+ ]
+ }
+
+
+# ---------------------------------------------------------------------------
+# Route registration
+# ---------------------------------------------------------------------------
+
def _register_routes(r: Router | NinjaAPI, ranker: HeuristicRanker):
"""Register search routes on the given router or API instance."""
@r.get(
"",
+ response=list[SearchResult],
summary="Search",
description=(
- "Search the Mwmbl index and return formatted results. "
- "Results are ranked using a heuristic ranker and filtered to a minimum relevance score. "
- "Pass the search query as the `s` query parameter."
+ "Search the Mwmbl index and return formatted results.\n\n"
+ "Results are ranked using a heuristic ranker that considers title, extract, "
+ "domain authority, and query-term match quality. "
+ "Each result's `title` and `extract` are returned as a list of text segments "
+ "where matching query terms are flagged with `is_bold: true` for easy "
+ "client-side highlighting.\n\n"
+ "The `source` field indicates where the result originated:\n"
+ "- `mwmbl` — organically crawled by the Mwmbl crawler\n"
+ "- `wikipedia` — sourced from Wikipedia\n"
+ "- `google` — originally suggested via Google\n"
+ "- `user` — submitted directly by a user\n\n"
+ "**Query parameter:** `s` — the search query string (required)."
),
+ openapi_extra={
+ "parameters": [
+ {
+ "name": "s",
+ "in": "query",
+ "required": True,
+ "description": "The search query string.",
+ "schema": {"type": "string", "example": "python tutorial"},
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "A list of formatted search results.",
+ "content": {
+ "application/json": {
+ "example": [
+ {
+ "url": "https://docs.python.org/3/tutorial/",
+ "title": [
+ {"value": "The ", "is_bold": False},
+ {"value": "Python", "is_bold": True},
+ {"value": " Tutorial — Python 3 documentation", "is_bold": False},
+ ],
+ "extract": [
+ {"value": "Python", "is_bold": True},
+ {"value": " is an easy to learn, powerful programming language.", "is_bold": False},
+ ],
+ "source": "mwmbl",
+ },
+ {
+ "url": "https://wiki.python.org/moin/BeginnersGuide/Programmers",
+ "title": [
+ {"value": "Python", "is_bold": True},
+ {"value": " for Programmers — ", "is_bold": False},
+ {"value": "Python", "is_bold": True},
+ {"value": " Wiki", "is_bold": False},
+ ],
+ "extract": [
+ {"value": "A list of ", "is_bold": False},
+ {"value": "Python", "is_bold": True},
+ {"value": " resources for experienced programmers.", "is_bold": False},
+ ],
+ "source": "mwmbl",
+ },
+ ]
+ }
+ },
+ }
+ },
+ },
)
def search(request, s: str):
- """
- Example request:
- GET /api/v1/search/?s=python+tutorial
-
- Example response:
- [
- {
- "url": "https://docs.python.org/3/tutorial/",
- "title": "The Python Tutorial",
- "extract": "Python is an easy to learn, powerful programming language...",
- "score": 0.92
- }
- ]
- """
results = ranker.search(s, [])
return [format_result(result, s) for result in results]
@@ -50,49 +196,131 @@ def _register_routes(r: Router | NinjaAPI, ranker: HeuristicRanker):
"/complete",
summary="Autocomplete",
description=(
- "Return autocomplete suggestions for a partial query string. "
+ "Return autocomplete suggestions for a partial query string.\n\n"
"Useful for powering search-as-you-type interfaces. "
- "Pass the partial query as the `q` query parameter."
+ "The response is a two-element list:\n"
+ "1. The original query string (echoed back).\n"
+ "2. A list of suggestion strings. Each suggestion is either:\n"
+ " - A completed query term (e.g. `\"python tutorial\"`).\n"
+ " - A direct URL prefixed with `\"go: \"` when the partial query "
+ "matches a URL in the index (e.g. `\"go: docs.python.org/3/tutorial\"`).\n"
+ " - A Google search suggestion prefixed with `\"search: google.com \"` "
+ "when no index results are found.\n\n"
+ "**Query parameter:** `q` — the partial query string (required)."
),
+ openapi_extra={
+ "parameters": [
+ {
+ "name": "q",
+ "in": "query",
+ "required": True,
+ "description": "The partial query string to complete.",
+ "schema": {"type": "string", "example": "pyth"},
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": (
+ "A two-element list: the original query string and a list of suggestions."
+ ),
+ "content": {
+ "application/json": {
+ "examples": {
+ "with_index_results": {
+ "summary": "Suggestions when index results exist",
+ "value": [
+ "pyth",
+ [
+ "go: docs.python.org/3/tutorial",
+ "python",
+ "python tutorial",
+ "python documentation",
+ ],
+ ],
+ },
+ "no_index_results": {
+ "summary": "Fallback to Google suggestions when no index results",
+ "value": [
+ "xyzzy lang",
+ [
+ "search: google.com xyzzy lang",
+ "search: google.com xyzzy language",
+ ],
+ ],
+ },
+ }
+ }
+ },
+ }
+ },
+ },
)
def complete(request, q: str):
- """
- Example request:
- GET /api/v1/search/complete?q=pyth
-
- Example response:
- ["python", "python tutorial", "python documentation"]
- """
return ranker.complete(q)
@r.get(
"/raw",
+ response=RawSearchResponse,
summary="Raw search results",
description=(
- "Return raw, unformatted search results directly from the index. "
- "Includes internal scoring fields useful for debugging and evaluation. "
- "Pass the search query as the `s` query parameter."
+ "Return raw, unformatted search results directly from the index.\n\n"
+ "Unlike the main `/search/` endpoint, results are **not** re-ranked or "
+ "formatted for display. The response includes internal scoring fields "
+ "that are useful for debugging, evaluation, and building custom ranking "
+ "pipelines.\n\n"
+ "Field descriptions for each result:\n"
+ "- `title` — page title as stored in the index.\n"
+ "- `url` — canonical URL of the page.\n"
+ "- `extract` — short text snippet from the page body.\n"
+ "- `score` — raw relevance score assigned during indexing (may be `null`).\n"
+ "- `term` — the index term under which this document was stored (may be `null`).\n"
+ "- `state` — integer representing the document's curation state "
+ "(see `DocumentState` enum; `null` means a standard organic result).\n\n"
+ "**Query parameter:** `s` — the search query string (required)."
),
- )
- def raw(request, s: str):
- """
- Example request:
- GET /api/v1/search/raw?s=python+tutorial
-
- Example response:
- {
- "query": "python tutorial",
- "results": [
+ openapi_extra={
+ "parameters": [
{
- "url": "https://docs.python.org/3/tutorial/",
- "title": "The Python Tutorial",
- "extract": "Python is an easy to learn...",
- "score": 0.92,
- "term": "python tutorial"
+ "name": "s",
+ "in": "query",
+ "required": True,
+ "description": "The search query string.",
+ "schema": {"type": "string", "example": "python tutorial"},
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "The raw query and unranked index results.",
+ "content": {
+ "application/json": {
+ "example": {
+ "query": "python tutorial",
+ "results": [
+ {
+ "title": "The Python Tutorial",
+ "url": "https://docs.python.org/3/tutorial/",
+ "extract": "Python is an easy to learn, powerful programming language.",
+ "score": 0.92,
+ "term": "python tutorial",
+ "state": None,
+ },
+ {
+ "title": "Python for Beginners",
+ "url": "https://www.python.org/about/gettingstarted/",
+ "extract": "Are you new to programming? Python is great for beginners.",
+ "score": 0.74,
+ "term": "python",
+ "state": None,
+ },
+ ],
+ }
+ }
+ },
}
- ]
- }
- """
+ },
+ },
+ )
+ def raw(request, s: str):
results = ranker.get_raw_results(s)
return {"query": s, "results": [dataclasses.asdict(result) for result in results]}