Skip to content

Official Codebase of 3DCodeVerse

Python
15.3 KB
Raw Blame
"""`3dcode-data` CLI — contributor-side commands to validate and upload (code -> 3D) projects.

    3dcode-data config set --endpoint ... --access-key-id ... --secret-access-key ...
    3dcode-data validate ./data
    3dcode-data push ./data --source yourname
"""

from __future__ import annotations

import json
from pathlib import Path

import typer

from . import __version__, registry
from .config import load_config, save_config
from .hashing import project_hashes
from .schema import META_NAME, find_projects, merge_preserved, validate_project

app = typer.Typer(add_completion=False, help="3DCodeVerse data toolkit.")
config_app = typer.Typer(help="Manage R2 endpoint / token / default source.")
app.add_typer(config_app, name="config")


@app.command()
def version():
    """Print the toolkit version."""
    typer.echo(f"3dcode-data {__version__}")


@config_app.command("set")
def config_set(
    endpoint: str = typer.Option(None),
    bucket: str = typer.Option(None),
    access_key_id: str = typer.Option(None),
    secret_access_key: str = typer.Option(None),
    source: str = typer.Option(None, help="default source folder for your uploads"),
    api: str = typer.Option(None, help="registry API base (default 3dcodebench.com)"),
    contrib_token: str = typer.Option(None, help="shared contributor token for the registry"),
    blender_5_0: str = typer.Option(None, help="local Blender 5.0 binary (for 3dcode-data exec)"),
    blender_5_1: str = typer.Option(None, help="local Blender 5.1 binary"),
    openscad: str = typer.Option(None, help="local OpenSCAD binary/AppImage (.scad)"),
    freecadcmd: str = typer.Option(None, help="local freecadcmd binary (FreeCAD)"),
    core_dir: str = typer.Option(None, help="admin canonical store dir (for 3dcode-data ingest)"),
):
    """Write R2 settings to ~/.config/3dcode/config.toml (chmod 600)."""
    path = save_config({
        "endpoint": endpoint, "bucket": bucket, "access_key_id": access_key_id,
        "secret_access_key": secret_access_key, "source": source,
        "api": api, "contrib_token": contrib_token,
        "blender_5_0": blender_5_0, "blender_5_1": blender_5_1,
        "openscad": openscad, "freecadcmd": freecadcmd, "core_dir": core_dir,
    })
    typer.echo(f"saved -> {path}")


@config_app.command("show")
def config_show():
    """Show the active config (secret masked)."""
    c = load_config()
    secret = (c.secret_access_key[:4] + "…") if c.secret_access_key else "(unset)"
    for k, v in {"endpoint": c.endpoint, "bucket": c.bucket,
                 "access_key_id": c.access_key_id or "(unset)",
                 "secret_access_key": secret, "source": c.source or "(unset)"}.items():
        typer.echo(f"  {k:18} {v}")


def _iter(root: Path, source: str):
    projects = find_projects(root)
    if not projects:
        typer.echo(f"no projects found under {root}", err=True)
        raise typer.Exit(1)
    for d in projects:
        errors, warnings, meta = validate_project(d, source)
        yield d, errors, warnings, meta


@app.command()
def validate(path: Path = typer.Argument(..., exists=True),
             source: str = typer.Option(None, help="source name (defaults to config)")):
    """Validate project(s) locally — no upload."""
    source = source or load_config().source or "unknown"
    bad = 0
    for d, errors, warnings, _ in _iter(path, source):
        if errors:
            bad += 1
            typer.secho(f"✗ {d.name}: {'; '.join(errors)}", fg="red")
        else:
            tag = typer.style("✓", fg="green")
            typer.echo(f"{tag} {d.name}" + (f"  ⚠ {'; '.join(warnings)}" if warnings else ""))
    if bad:
        raise typer.Exit(1)


@app.command()
def check(path: Path = typer.Argument(..., exists=True),
          source: str = typer.Option(None, help="source name (defaults to config)")):
    """Validate + fingerprint locally, and flag duplicates vs the existing corpus — no upload."""
    cfg = load_config()
    source = source or cfg.source or "unknown"
    index = registry.fetch_index(cfg)
    n = dups = 0
    for d, errors, warnings, meta in _iter(path, source):
        n += 1
        if errors:
            typer.secho(f"✗ {d.name}: {'; '.join(errors)}", fg="red")
            continue
        meta["dedup"] = project_hashes(d, meta)
        hits = registry.dedup_matches(meta, index)
        if hits:
            dups += 1
            for ex, why in hits:
                typer.secho(f"⚠ {d.name}: {why} as {ex}", fg="yellow")
        else:
            tail = f"  ⚠ {'; '.join(warnings)}" if warnings else ""
            typer.secho(f"✓ {d.name}: unique{tail}", fg="green")
    typer.echo(f"\nchecked {n} · {dups} with potential duplicates · index size {len(index)}")


@app.command()
def anomalies(path: Path = typer.Argument(..., exists=True),
              source: str = typer.Option(None, help="source name (defaults to config)")):
    """Distribution-based outlier scan — flag projects whose code length / char count is an outlier."""
    source = source or load_config().source or "unknown"
    rows = []
    for d, errors, _, meta in _iter(path, source):
        if errors or not meta:
            continue
        chars = len((d / meta["code"]["file"]).read_text(errors="replace"))
        rows.append((d.name, meta["code"]["lines"], chars))
    if len(rows) < 4:
        typer.echo(f"need >= 4 valid projects for a distribution (got {len(rows)})")
        return

    def bounds(vals: list[int]) -> tuple[float, float]:
        s = sorted(vals)
        q1, q3 = s[len(s) // 4], s[(3 * len(s)) // 4]
        iqr = q3 - q1
        return q1 - 1.5 * iqr, q3 + 1.5 * iqr

    lo_l, hi_l = bounds([r[1] for r in rows])
    lo_c, hi_c = bounds([r[2] for r in rows])
    flagged = 0
    for name, lines, chars in rows:
        why = []
        if lines < lo_l or lines > hi_l:
            why.append(f"lines={lines} (typical {int(max(0, lo_l))}–{int(hi_l)})")
        if chars < lo_c or chars > hi_c:
            why.append(f"chars={chars} (typical {int(max(0, lo_c))}–{int(hi_c)})")
        if why:
            flagged += 1
            typer.secho(f"⚠ {name}: {'; '.join(why)}", fg="yellow")
    typer.echo(f"\nscanned {len(rows)} · {flagged} length/char outliers")


@app.command("exec")
def exec_run(path: Path = typer.Argument(..., exists=True),
             source: str = typer.Option(None, help="source name (defaults to config)"),
             dialect: str = typer.Option(None, help="override code dialect (e.g. cadquery)"),
             write: bool = typer.Option(True, help="write the result into each meta.json")):
    """Run each project's code in its dialect runtime(s); record pass/fail in meta.json.

    Runs LOCALLY in your own runtimes (configure Blender via `3dcode-data config set
    --blender-5-0 ... --blender-5-1 ...`); nothing is uploaded. Pass = error-free AND a non-empty mesh.
    """
    from .dialects import run_dialect
    cfg = load_config()
    source = source or cfg.source or "unknown"
    npass = nfail = 0
    for d, errors, _, meta in _iter(path, source):
        if errors:
            typer.secho(f"✗ {d.name}: {'; '.join(errors)}", fg="red")
            continue
        merge_preserved(d, meta)                     # don't clobber license / provenance
        if dialect:
            meta["dialect"] = dialect
        res = run_dialect(meta["dialect"], d, meta, cfg)
        meta["exec"] = res
        if write:
            (d / META_NAME).write_text(json.dumps(meta, indent=2, ensure_ascii=False))
        parts, passed = [], True
        for rt, r in res.items():
            st = r.get("status")
            extra = ""
            if st == "ok":
                if r.get("verts"):
                    extra = f"({r['verts']}v)"
                elif r.get("volume") is not None:
                    extra = f"(vol {r['volume']})"
            parts.append(f"{rt}={st}{extra}")
            if st not in ("ok", "n/a", "unsupported"):
                passed = False
        typer.secho(f"{'✓' if passed else '✗'} {d.name}: {'  '.join(parts)}",
                    fg="green" if passed else "red")
        npass += passed
        nfail += not passed
    typer.echo(f"\nexec: {npass} passed · {nfail} failed")


@app.command()
def push(path: Path = typer.Argument(..., exists=True),
         source: str = typer.Option(None, help="source name (defaults to config)"),
         owner: str = typer.Option(None, help="who is responsible for this data"),
         dialect: str = typer.Option(None, help="override code dialect"),
         dry_run: bool = typer.Option(False, "--dry-run", help="validate + hash, do not upload"),
         no_dedup: bool = typer.Option(False, "--no-dedup", help="skip the duplicate check"),
         no_register: bool = typer.Option(False, "--no-register", help="upload but don't register in the registry")):
    """Validate -> hash -> dedup-check -> write meta.json -> upload to R2 -> register."""
    cfg = load_config()
    source = source or cfg.source
    if not source:
        typer.secho("no --source and no default source in config", fg="red", err=True)
        raise typer.Exit(1)
    contributor = owner or source

    r2 = None
    if not dry_run:
        from .storage import R2  # imported lazily so validate/dry-run need no creds
        r2 = R2(cfg)
    index = [] if no_dedup else registry.fetch_index(cfg)

    up = skip = warn = dup = 0
    for d, errors, warnings, meta in _iter(path, source):
        if errors:
            skip += 1
            typer.secho(f"✗ {d.name}: {'; '.join(errors)} — skipped", fg="red")
            continue
        if dialect:
            meta["dialect"] = dialect
        merge_preserved(d, meta)                     # keep exec / license / provenance from disk
        meta["owner"] = owner or meta.get("owner", "")
        meta["contributor"] = contributor
        meta["dedup"] = project_hashes(d, meta)
        matches = registry.dedup_matches(meta, index)
        # validation report — travels with the project so the admin Inbox can review it
        meta["report"] = {
            "warnings": warnings,
            "duplicates": [{"of": ex, "why": why} for ex, why in matches],
            "exec": meta.get("exec"),
            "files": sorted(p.relative_to(d).as_posix() for p in d.rglob("*") if p.is_file()),
        }
        (d / META_NAME).write_text(json.dumps(meta, indent=2, ensure_ascii=False))

        for ex, why in matches:
            dup += 1
            typer.secho(f"⚠ {d.name}: {why} as {ex}", fg="yellow")
        if warnings:
            warn += 1
            typer.secho(f"⚠ {d.name}: {'; '.join(warnings)}", fg="yellow")
        if dry_run:
            up += 1
            typer.echo(f"  (dry-run) {meta['id']} ready — {len(meta['dedup'])} fingerprints")
            continue
        result = r2.upload_tree(d, f"{source}/{d.name}")
        up += 1
        typer.secho(f"↑ {meta['id']}  ({result.files} files, {result.bytes/1e6:.1f} MB)", fg="green")
        if not no_register:
            resp = registry.register(cfg, meta)
            if resp.get("error"):
                typer.secho(f"  ! registry: {resp['error']}", fg="red")

    typer.echo(f"\n{'validated' if dry_run else 'uploaded'}: {up} · dup-flagged: {dup} · warn: {warn} · skipped: {skip}")


@app.command()
def render(path: Path = typer.Argument(..., exists=True),
          source: str = typer.Option(None, help="source name (defaults to config)"),
          dialect: str = typer.Option(None, help="override code dialect (e.g. cadquery)"),
          mode: str = typer.Option("white", help="white | textured")):
    """Render each project's code LOCALLY into renders/ (white clay or textured) — no upload.

    Runs the code in your local Blender (config --blender-5-0) and writes view_00..03.png +
    thumb.png. Then `3dcode-data push` picks up the renders (modality + phash dedup + Inbox preview).
    """
    from .dialects import render_dialect
    cfg = load_config()
    source = source or cfg.source or "unknown"
    n = fail = 0
    for d, errors, _, meta in _iter(path, source):
        if errors:
            typer.secho(f"✗ {d.name}: {'; '.join(errors)}", fg="red"); continue
        if dialect:
            meta["dialect"] = dialect
        res = render_dialect(meta["dialect"], d, meta, cfg, mode)
        st = res.get("status")
        if st == "ok":
            n += 1
            typer.secho(f"🖼 {d.name}: {len(res.get('images', []))} views → renders/", fg="green")
        else:
            fail += 1
            typer.secho(f"✗ {d.name}: {st}" + (f" — {res['error']}" if res.get("error") else ""), fg="red")
    typer.echo(f"\nrendered: {n} · failed: {fail}")


@app.command()
def grid(path: Path = typer.Argument(..., exists=True),
         out: Path = typer.Option(None, help="output PNG (default <dir>/_grid.png)"),
         cols: int = typer.Option(6, help="columns")):
    """Compose a contact-sheet PNG of a source's project thumbnails (quick visual review)."""
    from .grid import make_grid
    projects = find_projects(path)
    if not projects:
        typer.secho(f"no projects under {path}", fg="red", err=True)
        raise typer.Exit(1)
    dest = out or (path / "_grid.png")
    try:
        n = make_grid(projects, dest, cols=cols)
    except ImportError:
        typer.secho("grid needs Pillow — install 3dcode[dedup]", fg="red", err=True)
        raise typer.Exit(1)
    if n == 0:
        typer.secho("no thumbnails found — run `3dcode-data render` first?", fg="yellow")
    else:
        typer.secho(f"grid: {n} thumbnails → {dest}", fg="green")


@app.command()
def ingest(project_id: str = typer.Argument(None, help="<source>/<project> to ingest (or use --all-approved)"),
           all_approved: bool = typer.Option(False, "--all-approved", help="ingest every approved project"),
           keep_staging: bool = typer.Option(False, "--keep-staging", help="don't delete from R2 after pulling")):
    """ADMIN (lab-side): pull approved projects R2 → core dir, mark ingested, clear staging."""
    cfg = load_config()
    from .storage import R2
    r2 = R2(cfg)
    core = Path(cfg.core_dir)

    if all_approved:
        ids = [a["id"] for a in registry.fetch_approved(cfg)]
    elif project_id:
        ids = [project_id]
    else:
        typer.secho("give a <source>/<project> or --all-approved", fg="red", err=True)
        raise typer.Exit(1)
    if not ids:
        typer.echo("nothing approved to ingest")
        return

    done = fail = 0
    for pid in ids:
        try:
            dest = core / pid                          # core/<source>/<project>
            res = r2.download_tree(pid, dest)
            if res.files == 0:
                typer.secho(f"✗ {pid}: nothing in staging", fg="red"); fail += 1; continue
            meta_path = dest / META_NAME
            if meta_path.exists():
                meta = json.loads(meta_path.read_text())
            else:
                src, _, proj = pid.partition("/")
                meta = {"id": pid, "source": src, "project": proj}
            meta["status"] = "ingested"
            resp = registry.register(cfg, meta)
            if resp.get("error"):
                typer.secho(f"  ! mark-ingested: {resp['error']}", fg="yellow")
            if not keep_staging:
                r2.delete_prefix(pid.rstrip("/") + "/")
            typer.secho(f"↓ {pid}  ({res.files} files, {res.bytes / 1e6:.1f} MB) → {dest}", fg="green")
            done += 1
        except Exception as e:
            typer.secho(f"✗ {pid}: {e}", fg="red"); fail += 1
    typer.echo(f"\ningested: {done} · failed: {fail}")


if __name__ == "__main__":
    app()