Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

13.9 KB
Raw Blame
"""Shared docs corpus, immutable generation, publication, and reader contract."""

import hashlib
import json
import logging
import re
import shutil
import sqlite3
import threading
import uuid
from datetime import datetime, timezone
from pathlib import Path

FORMAT_VERSION = 1
EMBEDDING = {"model": "all-MiniLM-L6-v2", "dimensions": 384}
WRITER_VERSIONS = {"lancedb": "0.37.1", "sentence-transformers": "5.7.0", "pyarrow": "25.0.1"}


def write_json(path, value):
    Path(path).write_text(json.dumps(value, sort_keys=True, indent=2), encoding="utf-8")


def digest(value):
    return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest()


def capture_corpus(pages, expected_urls, failed_urls=()):
    """Identify one completed crawl; a missing discovered page fails the build."""
    urls = [p["url"] for p in pages]
    if failed_urls or not urls or any(not u for u in urls) or len(urls) != len(set(urls)):
        raise ValueError("Incomplete crawl: failed, empty, or duplicate pages")
    if set(urls) != set(expected_urls):
        raise ValueError("Incomplete crawl: discovered URLs differ from captured pages")
    pages = sorted(pages, key=lambda p: p["url"])
    return {
        "format_version": FORMAT_VERSION,
        "source": "https://cua.ai/docs (links discovered from /docs)",
        "captured_at": datetime.now(timezone.utc).isoformat(),
        "corpus_id": digest(pages),
        "expected_urls": sorted(urls),
        "pages": pages,
    }


def load_corpus(path):
    corpus = json.loads(Path(path).read_text())
    checked = capture_corpus(corpus["pages"], corpus["expected_urls"])
    if (
        corpus.get("format_version") != FORMAT_VERSION
        or checked["corpus_id"] != corpus["corpus_id"]
    ):
        raise ValueError("Corpus identity mismatch")
    return corpus


def prepare_pages(corpus):
    from markdown_it import MarkdownIt

    parser = MarkdownIt()
    included, exclusions = [], []
    for page in corpus["pages"]:
        parts = []
        for token in parser.parse(page.get("markdown", "")):
            if token.type in ("fence", "code_block"):
                parts.append(token.content)
            elif token.type == "inline":
                parts.append(
                    " ".join(
                        c.content for c in token.children or [] if c.type in ("text", "code_inline")
                    )
                )
        text = "\n\n".join(parts).strip()
        if not text:
            exclusions.append({"url": page["url"], "reason": "no searchable text"})
            continue
        included.append({**page, "text": text})
    if not included:
        raise ValueError("Corpus has no searchable pages")
    return included, exclusions


def make_chunks(pages, build_id):
    """Keep short pages and short paragraphs; both indexes cover identical URLs."""
    return [
        {
            "text": page["text"][offset : offset + 1000],
            "url": page["url"],
            "title": page.get("title") or "Untitled",
            "category": page.get("path_info", {}).get("category") or "unknown",
            "chunk_index": index,
            "build_id": build_id,
        }
        for page in pages
        for index, offset in enumerate(range(0, len(page["text"]), 800))
    ]


def write_sqlite(path, pages, build_id, corpus):
    with sqlite3.connect(path) as conn:
        conn.executescript("""
            CREATE TABLE pages (id INTEGER PRIMARY KEY, url TEXT UNIQUE NOT NULL,
                title TEXT, category TEXT, content TEXT, build_id TEXT NOT NULL,
                description TEXT, subcategory TEXT, page_name TEXT, raw_markdown TEXT);
            CREATE VIRTUAL TABLE pages_fts USING fts5(content, url UNINDEXED,
                title UNINDEXED, category UNINDEXED, content='pages', content_rowid='id');
            CREATE TABLE index_metadata (build_id TEXT NOT NULL, corpus_id TEXT NOT NULL,
                captured_at TEXT NOT NULL, source TEXT NOT NULL);
        """)
        conn.execute(
            "INSERT INTO index_metadata VALUES (?,?,?,?)",
            (build_id, corpus["corpus_id"], corpus["captured_at"], corpus["source"]),
        )
        conn.executemany(
            "INSERT INTO pages (url,title,category,content,build_id,description,subcategory,page_name,raw_markdown) "
            "VALUES (?,?,?,?,?,?,?,?,?)",
            [
                (
                    p["url"],
                    p.get("title") or "Untitled",
                    p.get("path_info", {}).get("category") or "unknown",
                    p["text"],
                    build_id,
                    p.get("description", ""),
                    p.get("path_info", {}).get("subcategory"),
                    p.get("path_info", {}).get("page", ""),
                    p.get("markdown", ""),
                )
                for p in pages
            ],
        )
        conn.execute("INSERT INTO pages_fts(pages_fts) VALUES ('rebuild')")


def write_vectors(path, chunks):
    from importlib.metadata import version

    for package, expected in WRITER_VERSIONS.items():
        if version(package) != expected:
            raise RuntimeError(
                f"Use the docs-mcp-server locked environment: {package}=={expected} required"
            )
    import lancedb
    from lancedb.embeddings import get_registry
    from lancedb.pydantic import LanceModel, Vector

    model = get_registry().get("sentence-transformers").create(name=EMBEDDING["model"])

    class DocChunk(LanceModel):
        text: str = model.SourceField()
        vector: Vector(model.ndims()) = model.VectorField()
        url: str
        title: str
        category: str
        chunk_index: int
        build_id: str

    table = lancedb.connect(path).create_table("docs", schema=DocChunk)
    for offset in range(0, len(chunks), 100):
        table.add(chunks[offset : offset + 100])


def open_vectors(path):
    import lancedb

    return lancedb.connect(path).open_table("docs")


def validate_pair(path, manifest, table):
    conn = sqlite3.connect(
        f"{(Path(path) / 'docs.sqlite').as_uri()}?mode=ro", uri=True, check_same_thread=False
    )
    try:
        expected = set(manifest["included_urls"])
        if expected | {e["url"] for e in manifest["exclusions"]} != set(manifest["expected_urls"]):
            raise ValueError("Manifest corpus coverage mismatch")
        metadata = conn.execute(
            "SELECT build_id,corpus_id,captured_at,source FROM index_metadata"
        ).fetchall()
        if metadata != [
            (
                manifest["build_id"],
                manifest["corpus_id"],
                manifest["captured_at"],
                manifest["source"],
            )
        ]:
            raise ValueError("SQLite build identity mismatch")
        rows = conn.execute("SELECT url, build_id FROM pages").fetchall()
        vectors = (
            table.search().select(["url", "build_id"]).limit(manifest["chunk_count"] + 1).to_list()
        )
        if (
            {r[0] for r in rows} != expected
            or {r["url"] for r in vectors} != expected
            or len(rows) != len(expected)
            or len(vectors) != manifest["chunk_count"]
            or any(r[1] != manifest["build_id"] for r in rows)
            or any(r["build_id"] != manifest["build_id"] for r in vectors)
        ):
            raise ValueError("SQLite/vector corpus or build identity mismatch")
        conn.row_factory = sqlite3.Row
        return conn
    except Exception:
        conn.close()
        raise


def file_hash(path):
    with Path(path).open("rb") as stream:
        return hashlib.file_digest(stream, "sha256").hexdigest()


def build_generation(corpus, root, vector_writer=write_vectors, vector_opener=open_vectors):
    """Build without touching the active generation. Return only a verified pair."""
    checked = capture_corpus(corpus["pages"], corpus["expected_urls"])
    if checked["corpus_id"] != corpus["corpus_id"]:
        raise ValueError("Corpus identity mismatch")
    pages, exclusions = prepare_pages(corpus)
    build_id = uuid.uuid4().hex
    path = Path(root).resolve() / "generations" / build_id
    path.mkdir(parents=True)
    chunks = make_chunks(pages, build_id)
    write_sqlite(path / "docs.sqlite", pages, build_id, corpus)
    vector_writer(path, chunks)
    manifest = {
        "format_version": FORMAT_VERSION,
        "build_id": build_id,
        "corpus_id": corpus["corpus_id"],
        "source": corpus["source"],
        "captured_at": corpus["captured_at"],
        "embedding": EMBEDDING,
        "writer_versions": WRITER_VERSIONS,
        "expected_urls": corpus["expected_urls"],
        "included_urls": [p["url"] for p in pages],
        "exclusions": exclusions,
        "chunk_count": len(chunks),
    }
    validate_pair(path, manifest, vector_opener(path)).close()
    manifest["files"] = {
        p.relative_to(path).as_posix(): file_hash(p) for p in sorted(path.rglob("*")) if p.is_file()
    }
    write_json(path / "manifest.json", manifest)
    return path, manifest


def verify_files(path, manifest):
    if manifest.get("format_version") != FORMAT_VERSION or manifest.get("embedding") != EMBEDDING:
        raise ValueError("Unsupported docs generation")
    if not manifest.get("files") or "docs.sqlite" not in manifest["files"]:
        raise ValueError("Incomplete docs generation manifest")
    actual = {
        p.relative_to(path).as_posix()
        for p in Path(path).rglob("*")
        if p.is_file() and p != Path(path) / "manifest.json"
    }
    if actual != set(manifest["files"]):
        raise ValueError("Incomplete docs generation: file inventory mismatch")
    for name, sha in manifest["files"].items():
        file = Path(path) / name
        if (
            file.resolve().parent != Path(path).resolve()
            and Path(path).resolve() not in file.resolve().parents
        ):
            raise ValueError("Invalid generation file path")
        if not file.is_file() or file_hash(file) != sha:
            raise ValueError(f"Incomplete docs generation: {name}")


def pointer_for(manifest):
    return {"build_id": manifest["build_id"], "manifest_sha256": digest(manifest)}


def activate(root, manifest):
    """Local filesystems: replace the small pointer only after validation/copy."""
    root = Path(root)
    verify_files(root / "generations" / manifest["build_id"], manifest)
    staging = root / f".current-{uuid.uuid4().hex}.json"
    write_json(staging, pointer_for(manifest))
    staging.replace(root / "current.json")


def publish_generation(s3, bucket, path, manifest):
    """Immutable objects first, then one commit marker. Never delete old builds."""
    verify_files(path, manifest)
    if json.loads((Path(path) / "manifest.json").read_text()) != manifest:
        raise ValueError("Publication manifest mismatch")
    prefix = f"docs_db/generations/{manifest['build_id']}/"
    for name in sorted(manifest["files"]):
        s3.upload_file(str(Path(path) / name), bucket, prefix + name)
    s3.upload_file(str(Path(path) / "manifest.json"), bucket, prefix + "manifest.json")
    s3.put_object(
        Bucket=bucket,
        Key="docs_db/current.json",
        Body=json.dumps(pointer_for(manifest)).encode(),
        ContentType="application/json",
    )
    return len(manifest["files"]) + 2


def copy_generation(source, destination):
    """Copy a committed generation to local disk, then activate it atomically."""
    source, destination = Path(source), Path(destination)
    pointer = json.loads((source / "current.json").read_text())
    build_id = pointer["build_id"]
    if not re.fullmatch(r"[a-f0-9]{32}", build_id):
        raise ValueError("Invalid docs build id")
    if (destination / "current.json").exists():
        if json.loads((destination / "current.json").read_text()) == pointer:
            return
    origin = source / "generations" / build_id
    manifest = json.loads((origin / "manifest.json").read_text())
    if pointer != pointer_for(manifest):
        raise ValueError("Copy manifest identity mismatch")
    verify_files(origin, manifest)
    target = destination / "generations" / build_id
    shutil.copytree(origin, target, dirs_exist_ok=True)
    activate(destination, manifest)


class GenerationReader:
    """Refresh both handles together; reject bad candidates and keep last good."""

    def __init__(self, root, vector_opener=open_vectors, refresh=None):
        self.root = Path(root).resolve()
        self.vector_opener = vector_opener
        self.current = None
        self.error = None
        self.lock = threading.RLock()
        self.refresh = refresh

    def get(self):
        with self.lock:
            try:
                if self.refresh:
                    self.refresh()
                pointer = json.loads((self.root / "current.json").read_text())
                build_id = pointer["build_id"]
                if not re.fullmatch(r"[a-f0-9]{32}", build_id):
                    raise ValueError("Invalid docs build id")
                if self.current and pointer == pointer_for(self.current[2]):
                    self.error = None
                    return self.current
                path = self.root / "generations" / build_id
                manifest = json.loads((path / "manifest.json").read_text())
                if (
                    manifest["build_id"] != build_id
                    or digest(manifest) != pointer["manifest_sha256"]
                ):
                    raise ValueError("Manifest identity mismatch")
                verify_files(path, manifest)
                table = self.vector_opener(path)
                conn = validate_pair(path, manifest, table)
                # Requests retain their returned handle while a new generation opens.
                self.current = (conn, table, manifest)
                self.error = None
            except Exception as error:
                self.error = str(error)
                logging.warning("Docs generation refresh rejected: %s", error)
                if self.current is None:
                    raise RuntimeError("No verified docs generation available") from error
            return self.current