Skip to content

Official Codebase of 3DCodeVerse

Python

models: the function-calling surface of the three clients goes with the api-agent it served

gaoypeng6d56f8266ced
diff --git a/harness/bench/cost_report.py b/harness/bench/cost_report.py
deleted file mode 100644
index 12c9fabf..00000000
--- a/harness/bench/cost_report.py
+++ /dev/null
@@ -1,51 +0,0 @@
-"""Battery cost report: ``python bench/cost_report.py bench/out/<battery> [...]``.
-
-Writes ``cost_report.md`` next to the battery (or ``--out``) and prints the
-summary. Same audit as ``3dcv cost``; this entry point exists so a battery can
-be costed from a script without the CLI, and so the report lands beside
-``report.md`` / ``report.html``.
-
- python bench/cost_report.py bench/out/static_v1_flash
- python bench/cost_report.py runs bench/out/*_flash --out bench/out/cost_all.md --recheck
-"""
-
-from __future__ import annotations
-
-import argparse
-import sys
-from pathlib import Path
-
-from codeverse.cost import audit_runs
-from codeverse.cost.report import console as text_report
-from codeverse.cost.report import markdown, runs_table
-
-
-def main(argv: list[str] | None = None) -> int:
- ap = argparse.ArgumentParser(description="Cost report for one or more run trees.")
- ap.add_argument("paths", nargs="+", type=Path, help="run dirs or trees (bench/out/<battery>, runs/)")
- ap.add_argument("--out", type=Path, default=None,
- help="markdown output (default: <first path>/cost_report.md)")
- ap.add_argument("--recheck", action="store_true",
- help="re-price every call with today's table instead of what was billed")
- ap.add_argument("--title", default="", help="report title")
- ap.add_argument("--quiet", action="store_true")
- args = ap.parse_args(argv)
-
- audit = audit_runs(args.paths, recheck=args.recheck)
- if not audit.runs:
- print(f"no runs found under {', '.join(str(p) for p in args.paths)}", file=sys.stderr)
- return 2
- title = args.title or f"Cost report — {', '.join(p.name for p in args.paths)}"
- out = args.out or (args.paths[0] / "cost_report.md" if args.paths[0].is_dir() else Path("cost_report.md"))
- out.parent.mkdir(parents=True, exist_ok=True)
- out.write_text(markdown(audit, title=title))
- if not args.quiet:
- print(text_report(audit))
- print()
- print(runs_table(audit, limit=15))
- print(f"\nwrote {out}")
- return 0
-
-
-if __name__ == "__main__": # pragma: no cover
- raise SystemExit(main())
diff --git a/harness/bench/refs_ab_readout.py b/harness/bench/refs_ab_readout.py
deleted file mode 100644
index 77ff9999..00000000
--- a/harness/bench/refs_ab_readout.py
+++ /dev/null
@@ -1,100 +0,0 @@
-"""Read out a reference-photo A/B: judge BOTH arms offline with the same LikenessJudge.
-
- python bench/refs_ab_readout.py bench/out/refs_v1_graphics --refs bench/refs/tsr_gfx_aurora_ridge \
- --rubric shader_v2 --model gemini:gemini-3.1-pro-preview --n 2
-
-The in-loop judge differs by arm on purpose (the arm with photos is judged by LikenessJudge,
-the arm without by the plain rubric judge), so loop-time scores cannot be paired. This script
-takes each run's best round (sheet, frames, gates, plan digest, frame metrics — what the loop
-hands the judge), attaches the SAME reference photos to every run's spec, and judges all of
-them with one LikenessJudge. Arms are read from the prompt id (``<name>_ref_<seed>`` /
-``<name>_noref_<seed>``); the readout pairs seeds and prints per-run scores, defects, and the
-paired mean delta. Verdicts are cached under ``<out>/refs_ab_judge/<id>.json``.
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import statistics as st
-import sys
-from pathlib import Path
-from typing import Any
-
-sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) # `python bench/refs_ab_readout.py` from the repo root
-
-from bench.judge_calib_graphics import best_round # noqa: E402
-from codeverse.contracts.artifacts import GateReport, RenderSet # noqa: E402
-from codeverse.contracts.spec import ReferenceImage, Spec
-from codeverse.judges.base import JudgeInput, plan_digest
-from codeverse.judges.vlm_judge import LikenessJudge
-from codeverse.tracks.graphics import frame_stats_text
-from codeverse.workspace import Workspace
-
-SUFFIXES = (".png", ".jpg", ".jpeg", ".webp")
-
-
-def judge_run(run: Path, refs: list[Path], judge: LikenessJudge, cache: Path) -> dict[str, Any]:
- cached = cache / f"{run.name}.json"
- if cached.is_file():
- return json.loads(cached.read_text())
- rnd = best_round(run)
- if rnd is None:
- return {"id": run.name, "error": "no judged round"}
- spec = Spec.model_validate(json.loads((run / "spec.json").read_text()))
- spec = spec.model_copy(update={"references": [ReferenceImage(path=str(p), role="likeness", note=p.stem.replace("_", " ")) for p in refs]})
- ws = Workspace(run)
- plan_path = run / "plan.json"
- last_index = max((int(f.stem[1:]) for f in run.glob("rounds/r*.json")), default=0)
- extra = ("FRAME METRICS (harness-measured):\n" + frame_stats_text(ws)) if int(rnd.get("index", 0)) == last_index else \
- "FRAME METRICS: not available for this round; judge motion from the frames."
- inp = JudgeInput(spec=spec, renders=RenderSet.model_validate(rnd["renders"]),
- gates=[GateReport.model_validate(g) for g in rnd.get("gates") or []],
- round_index=int(rnd.get("index", 0)),
- plan_summary=plan_digest(json.loads(plan_path.read_text())) if plan_path.is_file() else "", extra_context=extra)
- v = judge.judge(inp)
- raw = v.raw if isinstance(v.raw, dict) else json.loads(v.raw or "{}")
- row = {"id": run.name, "round": int(rnd.get("index", 0)), "overall": v.overall, "loop_overall": (rnd.get("judgment") or {}).get("overall"),
- "uncapped": raw.get("overall_uncapped"), "scores": v.scores,
- "defects": [d for d, on in (raw.get("defects") or {}).items() if on],
- "caps": [c.get("rule") for c in (raw.get("caps") or {}).get("caps_applied", [])], "summary": (v.summary or "")[:500]}
- cache.mkdir(parents=True, exist_ok=True)
- cached.write_text(json.dumps(row, indent=1))
- return row
-
-
-def main(argv: list[str] | None = None) -> int:
- ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
- ap.add_argument("out", help="bench/out/<battery> of the A/B")
- ap.add_argument("--refs", required=True, help="folder of the reference photos to attach to EVERY run")
- ap.add_argument("--rubric", default="shader_v2")
- ap.add_argument("--model", default="gemini:gemini-3.1-pro-preview")
- ap.add_argument("--n", type=int, default=2)
- a = ap.parse_args(argv)
- out = Path(a.out)
- refs = sorted(p for p in Path(a.refs).iterdir() if p.suffix.lower() in SUFFIXES)
- judge = LikenessJudge(a.model, n_samples=a.n, rubric=a.rubric)
- rows = [judge_run(run, refs, judge, out / "refs_ab_judge") for run in sorted((out / "runs").iterdir()) if (run / "spec.json").is_file()]
- ok = {r["id"]: r for r in rows if "overall" in r}
- print(f"# refs A/B readout — {len(ok)} runs judged with LikenessJudge({a.rubric}, {a.model}, n={a.n}), photos: {[p.name for p in refs]}\n")
- print("| run | round | loop score | same-judge score | defects | caps |\n|---|---|---|---|---|---|")
- for rid, r in sorted(ok.items()):
- print(f"| {rid} | r{r['round']:02d} | {r['loop_overall']:.3f} | **{r['overall']:.3f}** | {', '.join(r['defects'])} | {', '.join(c for c in r['caps'] if c)} |")
- seeds: dict[str, dict[str, float]] = {}
- for rid, r in ok.items():
- name, arm, seed = rid.rsplit("_", 2)
- seeds.setdefault(seed, {})[arm] = r["overall"]
- deltas = [s["ref"] - s["noref"] for s in seeds.values() if "ref" in s and "noref" in s]
- if deltas:
- m = st.mean(deltas)
- sd = st.stdev(deltas) if len(deltas) > 1 else float("nan")
- print(f"\npaired (ref − noref): n={len(deltas)} mean {m:+.3f} sd {sd:.3f} sign {sum(d > 0 for d in deltas)}/{len(deltas)}"
- f" (judged-score A/A floor sd 0.202)")
- for r in rows:
- if "error" in r:
- print(f"- {r['id']}: {r['error']}")
- return 0
-
-
-if __name__ == "__main__":
- raise SystemExit(main())
diff --git a/harness/codeverse/contracts/__init__.py b/harness/codeverse/contracts/__init__.py
index 8820a9b4..08280611 100644
--- a/harness/codeverse/contracts/__init__.py
+++ b/harness/codeverse/contracts/__init__.py
@@ -31,9 +31,6 @@ from codeverse.contracts.chat import (
ChatResponse,
ImagePart,
TextPart,
- ToolCallPart,
- ToolResultPart,
- ToolSpec,
)
from codeverse.contracts.common import (
ENTRY_FILE,
@@ -115,9 +112,6 @@ __all__ = [
"TRACK_INFO",
"TRACK_LANGUAGES",
"TextPart",
- "ToolCallPart",
- "ToolResultPart",
- "ToolSpec",
"Track",
"TrackInfo",
"Usage",
diff --git a/harness/codeverse/contracts/chat.py b/harness/codeverse/contracts/chat.py
index 44fbc6fa..8780528c 100644
--- a/harness/codeverse/contracts/chat.py
+++ b/harness/codeverse/contracts/chat.py
@@ -22,27 +22,11 @@ class ImagePart(BaseModel):
label: str = ""
-class ToolCallPart(BaseModel):
- type: Literal["tool_call"] = "tool_call"
- id: str
- name: str
- arguments: dict[str, Any] = Field(default_factory=dict)
-
-
-class ToolResultPart(BaseModel):
- type: Literal["tool_result"] = "tool_result"
- call_id: str
- name: str
- content: str
- images: list[ImagePart] = Field(default_factory=list)
- is_error: bool = False
-
-
-Part = TextPart | ImagePart | ToolCallPart | ToolResultPart
+Part = TextPart | ImagePart
class ChatMessage(BaseModel):
- role: Literal["user", "assistant", "tool"]
+ role: Literal["user", "assistant"]
parts: list[Part]
@classmethod
@@ -60,17 +44,10 @@ class ChatMessage(BaseModel):
return "\n".join(p.text for p in self.parts if isinstance(p, TextPart))
-class ToolSpec(BaseModel):
- name: str
- description: str
- parameters: dict[str, Any] = Field(description="JSON schema of the arguments object")
-
-
class ChatRequest(BaseModel):
messages: list[ChatMessage]
system: str = ""
response_schema: dict[str, Any] | None = Field(default=None, description="force JSON matching this schema")
- tools: list[ToolSpec] | None = None
temperature: float = 0.7
#: 65 536 is the declared output limit of every gemini-3.x model the harness uses
#: (models.get: input 1 048 576 / output 65 536). It is a CEILING, not a reservation —
@@ -92,7 +69,6 @@ class ChatRequest(BaseModel):
class ChatResponse(BaseModel):
text: str = ""
parsed: Any = None
- tool_calls: list[ToolCallPart] = Field(default_factory=list)
finish_reason: str = ""
usage: Usage = Field(default_factory=Usage)
raw: dict[str, Any] = Field(default_factory=dict)
diff --git a/harness/codeverse/models/anthropic.py b/harness/codeverse/models/anthropic.py
index a0961e44..bdb6e799 100644
--- a/harness/codeverse/models/anthropic.py
+++ b/harness/codeverse/models/anthropic.py
@@ -1,4 +1,4 @@
-"""ChatRequest ⇄ Anthropic Messages API dicts (messages, tools, thinking, parse).
+"""ChatRequest ⇄ Anthropic Messages API dicts (messages, submit tool, thinking, parse).
Model-family rules (Anthropic API, 2026-08):
* 4.6+ family (opus-5, sonnet-5, fable-5, mythos, opus-4-6/7/8, sonnet-4-6):
@@ -24,14 +24,10 @@ from codeverse.contracts.chat import (
ChatResponse,
ImagePart,
TextPart,
- ToolCallPart,
- ToolResultPart,
- ToolSpec,
)
from codeverse.contracts.common import Usage
from codeverse.models.base import ModelError
from codeverse.models.parts import (
- BoundedCache,
Stopwatch,
attempt_timeout_s,
classify_sdk_exception,
@@ -81,12 +77,6 @@ def sampling_allowed(model: str) -> bool:
return not any(tok in m for tok in _NO_SAMPLING)
-#: Assistant turns that contain tool calls must be replayed WITH their
-#: thinking blocks on the next request. ``ToolCallPart`` cannot carry them,
-#: so we remember the blocks under the first tool-call id (bounded LRU).
-THINKING_BLOCKS: BoundedCache[list[dict[str, Any]]] = BoundedCache(2048)
-
-
# ------------------------------------------------------------------ messages
def _image_block(p: ImagePart) -> dict[str, Any]:
data, mime = image_b64(p)
@@ -94,13 +84,11 @@ def _image_block(p: ImagePart) -> dict[str, Any]:
def to_messages(messages: list[ChatMessage]) -> list[dict[str, Any]]:
- """ChatMessages → Anthropic ``messages``. ``tool`` role → user turn of
- ``tool_result`` blocks; consecutive same-role turns are merged."""
+ """ChatMessages → Anthropic ``messages``; consecutive same-role turns are merged."""
out: list[dict[str, Any]] = []
for msg in messages:
role = "assistant" if msg.role == "assistant" else "user"
blocks: list[dict[str, Any]] = []
- first_call_id: str | None = None
for p in msg.parts:
if isinstance(p, TextPart):
if p.text:
@@ -109,31 +97,8 @@ def to_messages(messages: list[ChatMessage]) -> list[dict[str, Any]]:
if p.label:
blocks.append({"type": "text", "text": f"[image: {p.label}]"})
blocks.append(_image_block(p))
- elif isinstance(p, ToolCallPart):
- first_call_id = first_call_id or p.id
- blocks.append(
- {"type": "tool_use", "id": p.id, "name": p.name, "input": dict(p.arguments)}
- )
- elif isinstance(p, ToolResultPart):
- content: list[dict[str, Any]] = [
- {"type": "text", "text": p.content or "(no output)"}
- ]
- for img in p.images:
- if img.label:
- content.append({"type": "text", "text": f"[image: {img.label}]"})
- content.append(_image_block(img))
- block: dict[str, Any] = {
- "type": "tool_result",
- "tool_use_id": p.call_id,
- "content": content,
- }
- if p.is_error:
- block["is_error"] = True
- blocks.append(block)
if not blocks:
continue
- if role == "assistant" and first_call_id:
- blocks = list(THINKING_BLOCKS.get(first_call_id) or []) + blocks
if out and out[-1]["role"] == role:
out[-1]["content"].extend(blocks)
else:
@@ -146,13 +111,6 @@ def to_messages(messages: list[ChatMessage]) -> list[dict[str, Any]]:
# --------------------------------------------------------------------- tools
-def to_tool(t: ToolSpec) -> dict[str, Any]:
- schema = (
- to_anthropic_schema(t.parameters) if t.parameters else {"type": "object", "properties": {}}
- )
- return {"name": t.name, "description": t.description, "input_schema": schema}
-
-
def submit_tool(schema: dict[str, Any]) -> dict[str, Any]:
return {
"name": SUBMIT_TOOL,
@@ -189,7 +147,6 @@ def build_kwargs(request: ChatRequest, model: str, *, json_mode: str = "tool") -
if sampling_allowed(model) and not thinking_on:
kw["temperature"] = request.temperature
- tools = [to_tool(t) for t in request.tools] if request.tools else []
if request.response_schema is not None:
if json_mode == "output_config":
kw.setdefault("output_config", {})["format"] = {
@@ -197,8 +154,8 @@ def build_kwargs(request: ChatRequest, model: str, *, json_mode: str = "tool") -
"schema": to_anthropic_schema(request.response_schema),
}
else:
- tools.append(submit_tool(request.response_schema))
- if not request.tools and not thinking_on:
+ kw["tools"] = [submit_tool(request.response_schema)]
+ if not thinking_on:
kw["tool_choice"] = {"type": "tool", "name": SUBMIT_TOOL}
else:
# forced tool_choice is incompatible with thinking; nudge instead
@@ -206,38 +163,27 @@ def build_kwargs(request: ChatRequest, model: str, *, json_mode: str = "tool") -
(system + "\n\n" if system else "")
+ f"When you have the final answer, call the `{SUBMIT_TOOL}` tool exactly once with the complete JSON object."
)
- if tools:
- kw["tools"] = tools
if system:
kw["system"] = system
return kw
# ------------------------------------------------------------------ response
-def parse_content(content: list[Any]) -> tuple[str, list[ToolCallPart], Any, list[dict[str, Any]]]:
- """→ ``(text, tool_calls, submit_input, thinking_blocks)`` from response blocks."""
+def parse_content(content: list[Any]) -> tuple[str, Any]:
+ """→ ``(text, submit_input)`` from response blocks."""
texts: list[str] = []
- calls: list[ToolCallPart] = []
submit: Any = None
- thinking: list[dict[str, Any]] = []
for block in content:
btype = getattr(block, "type", None)
if btype == "text":
texts.append(block.text)
- elif btype == "tool_use":
- args = (
+ elif btype == "tool_use" and block.name == SUBMIT_TOOL:
+ submit = (
block.input
if isinstance(block.input, dict)
else json.loads(json.dumps(block.input))
)
- if block.name == SUBMIT_TOOL:
- submit = args
- else:
- calls.append(ToolCallPart(id=block.id, name=block.name, arguments=dict(args)))
- elif btype in ("thinking", "redacted_thinking"):
- dump = block.model_dump() if hasattr(block, "model_dump") else dict(block)
- thinking.append(dump)
- return "".join(texts), calls, submit, thinking
+ return "".join(texts), submit
# ===================================================================== anthropic
@@ -323,11 +269,9 @@ class AnthropicModel:
# the client is built once with a fixed timeout; the deadline is per call
msg = client.messages.create(
model=self.model, timeout=attempt_timeout_s(deadline, self.timeout_s), **kwargs)
- text, calls, submit, thinking_blocks = parse_content(msg.content)
+ text, submit = parse_content(msg.content)
stop = str(msg.stop_reason or "")
- if calls and thinking_blocks:
- THINKING_BLOCKS.put(calls[0].id, thinking_blocks)
- usage = self._usage(msg.usage, sw.ms, len(calls))
+ usage = self._usage(msg.usage, sw.ms)
if stop == "refusal":
raise ModelError(
f"Anthropic refused the request: {getattr(msg, 'stop_details', None)}",
@@ -345,7 +289,7 @@ class AnthropicModel:
parsed = strip_control_chars(submit)
if not text:
text = json.dumps(parsed)
- elif not calls:
+ else:
try:
parsed = parse_json_lenient(text)
except JsonParseError as exc:
@@ -353,15 +297,13 @@ class AnthropicModel:
f"structured output missing (stop_reason={stop}): {exc}",
retryable=stop != "max_tokens", usage=usage,
) from exc
- if not text and not calls and parsed is None:
+ if not text and parsed is None:
raise ModelError(f"Anthropic returned no content (stop_reason={stop})",
retryable=True, usage=usage)
raw: dict[str, Any] = {"stop_reason": stop, "id": msg.id, "model": msg.model}
- return ChatResponse(
- text=text, parsed=parsed, tool_calls=calls, finish_reason=stop, usage=usage, raw=raw
- )
+ return ChatResponse(text=text, parsed=parsed, finish_reason=stop, usage=usage, raw=raw)
- def _usage(self, u: Any, latency_ms: int, n_calls: int) -> Usage:
+ def _usage(self, u: Any, latency_ms: int) -> Usage:
cache_read = int(getattr(u, "cache_read_input_tokens", 0) or 0)
cache_write = int(getattr(u, "cache_creation_input_tokens", 0) or 0)
usage = Usage(
@@ -371,7 +313,6 @@ class AnthropicModel:
output_tokens=int(u.output_tokens or 0),
cached_tokens=cache_read,
thoughts_tokens=0, # thinking is billed inside output_tokens
- tool_calls=n_calls,
latency_ms=latency_ms,
)
usage.cost_usd = estimate_cost("anthropic", self.model, usage) + cache_write_surcharge(
diff --git a/harness/codeverse/models/base.py b/harness/codeverse/models/base.py
index 226780dd..2a6cd99e 100644
--- a/harness/codeverse/models/base.py
+++ b/harness/codeverse/models/base.py
@@ -23,7 +23,6 @@ class ChatModel(Protocol):
"""Blocking call. Must:
* honour ``request.response_schema`` (return ``parsed`` as a dict/list),
* attach images (``ImagePart``) inline in order,
- * return tool calls when ``request.tools`` is given and the model calls one,
* fill ``usage`` including ``cost_usd`` (see ``pricing.py``),
* raise ``ModelError`` (retryable=True/False) rather than provider exceptions.
"""
diff --git a/harness/codeverse/models/gemini.py b/harness/codeverse/models/gemini.py
index f730a3e5..31920e57 100644
--- a/harness/codeverse/models/gemini.py
+++ b/harness/codeverse/models/gemini.py
@@ -13,7 +13,6 @@
from __future__ import annotations
import io
-import json
import logging
import os
import re
@@ -35,13 +34,10 @@ from codeverse.contracts.chat import (
ChatResponse,
ImagePart,
TextPart,
- ToolCallPart,
- ToolResultPart,
- ToolSpec,
)
from codeverse.contracts.common import Usage
from codeverse.models.base import ModelError
-from codeverse.models.parts import BoundedCache, Stopwatch, image_bytes
+from codeverse.models.parts import Stopwatch, image_bytes
from codeverse.models.pricing import estimate_cost, per_image_usd
from codeverse.models.retry import (
MAX_WAIT_S,
@@ -67,40 +63,9 @@ RETRYABLE_FINISH = {"RECITATION", "MALFORMED_FUNCTION_CALL", "OTHER", "UNEXPECTE
FATAL_FINISH = {"SAFETY", "PROHIBITED_CONTENT", "BLOCKLIST", "SPII", "IMAGE_SAFETY"}
-#: Gemini 3 function calls carry an opaque ``thought_signature`` that MUST be
-#: echoed back with the call on the next turn. ``ToolCallPart`` has no slot for
-#: it, so we remember signatures by our synthetic call id (bounded LRU).
-SIGNATURES: BoundedCache[bytes] = BoundedCache(4096)
-_call_counter = [0]
-_call_lock = threading.Lock()
-
-#: prefix of the ids WE mint when the provider sent none (``extract_candidate``).
-#: A synthetic id exists only to pair ToolCallPart/ToolResultPart locally and is
-#: NEVER echoed back to the provider; a GENUINE provider id is echoed verbatim on
-#: both ``FunctionCall`` and ``FunctionResponse``.
-SYNTHETIC_CALL_ID_PREFIX = "synth-call-"
-
-
-def _next_call_id() -> str:
- with _call_lock:
- _call_counter[0] += 1
- return f"{SYNTHETIC_CALL_ID_PREFIX}{_call_counter[0]:06d}"
-
-
-def is_synthetic_call_id(call_id: str | None) -> bool:
- """True for ids minted here rather than issued by the provider."""
- return bool(call_id) and str(call_id).startswith(SYNTHETIC_CALL_ID_PREFIX)
-
-
-def _echo_id(call_id: str | None) -> str | None:
- """The id to send back to Gemini: the provider's own id verbatim, never a synthetic one."""
- return None if not call_id or is_synthetic_call_id(call_id) else str(call_id)
-
-
# ------------------------------------------------------------------ contents
def to_contents(messages: list[ChatMessage]) -> list[types.Content]:
- """ChatMessages → Gemini ``Content`` list. ``tool`` messages become a user
- turn of ``function_response`` parts (images attached as extra parts)."""
+ """ChatMessages → Gemini ``Content`` list."""
out: list[types.Content] = []
for msg in messages:
role = "model" if msg.role == "assistant" else "user"
@@ -114,23 +79,6 @@ def to_contents(messages: list[ChatMessage]) -> list[types.Content]:
if p.label:
parts.append(types.Part.from_text(text=f"[image: {p.label}]"))
parts.append(types.Part.from_bytes(data=raw, mime_type=mime))
- elif isinstance(p, ToolCallPart):
- part = types.Part(
- function_call=types.FunctionCall(
- name=p.name, args=dict(p.arguments), id=_echo_id(p.id)
- )
- )
- sig = SIGNATURES.get(p.id)
- if sig:
- part.thought_signature = sig
- parts.append(part)
- elif isinstance(p, ToolResultPart):
- parts.append(_function_response_part(p))
- for img in p.images:
- raw, mime = image_bytes(img)
- if img.label:
- parts.append(types.Part.from_text(text=f"[image: {img.label}]"))
- parts.append(types.Part.from_bytes(data=raw, mime_type=mime))
if not parts:
continue
# merge consecutive same-role turns (Gemini wants strict alternation)
@@ -143,43 +91,14 @@ def to_contents(messages: list[ChatMessage]) -> list[types.Content]:
return out
-def _function_response_part(p: ToolResultPart) -> types.Part:
- payload: dict[str, Any]
- try:
- parsed = json.loads(p.content)
- payload = parsed if isinstance(parsed, dict) else {"result": parsed}
- except (json.JSONDecodeError, TypeError):
- payload = {"result": p.content}
- if p.is_error:
- payload = {"error": payload.get("result", payload)} if "error" not in payload else payload
- return types.Part(
- function_response=types.FunctionResponse(name=p.name, response=payload, id=_echo_id(p.call_id))
- )
-
-
# -------------------------------------------------------------------- config
-def to_tools(tools: list[ToolSpec]) -> list[types.Tool]:
- decls = [
- types.FunctionDeclaration(
- name=t.name,
- description=t.description,
- parameters=to_gemini_schema(t.parameters) if t.parameters else None,
- )
- for t in tools
- ]
- return [types.Tool(function_declarations=decls)]
-
-
def build_config(
request: ChatRequest,
*,
timeout_ms: int,
use_thinking: bool = True,
- warnings: list[str] | None = None,
) -> types.GenerateContentConfig:
- """Map a ChatRequest onto ``GenerateContentConfig``. Function calling and
- JSON mode are mutually exclusive on Gemini: with tools, the schema is
- dropped (the text is still parsed leniently) and a warning is recorded."""
+ """Map a ChatRequest onto ``GenerateContentConfig``."""
cfg: dict[str, Any] = {
"temperature": request.temperature,
"max_output_tokens": request.max_output_tokens,
@@ -191,17 +110,7 @@ def build_config(
cfg["thinking_config"] = types.ThinkingConfig(
thinking_budget=THINKING_BUDGET[request.thinking], include_thoughts=False
)
- if request.tools:
- cfg["tools"] = to_tools(request.tools)
- cfg["tool_config"] = types.ToolConfig(
- function_calling_config=types.FunctionCallingConfig(mode="AUTO")
- )
- cfg["automatic_function_calling"] = types.AutomaticFunctionCallingConfig(disable=True)
- if request.response_schema is not None and warnings is not None:
- warnings.append(
- "response_schema ignored: Gemini cannot combine function calling with JSON mode"
- )
- elif request.response_schema is not None:
+ if request.response_schema is not None:
cfg["response_mime_type"] = "application/json"
cfg["response_schema"] = to_gemini_schema(request.response_schema)
return types.GenerateContentConfig(**cfg)
@@ -253,29 +162,19 @@ def _guard_candidates(resp: types.GenerateContentResponse, usage: Usage | None,
def extract_candidate(
resp: types.GenerateContentResponse, usage: Usage | None = None
-) -> tuple[str, list[ToolCallPart], str]:
- """→ ``(text, tool_calls, finish_reason)``. Raises ``ModelError`` on blocked
+) -> tuple[str, str]:
+ """→ ``(text, finish_reason)``. Raises ``ModelError`` on blocked
prompts (non-retryable) or empty candidates (retryable); ``usage`` (what the caller
already parsed off ``resp``) rides on the error, because every one of these
failures was billed."""
cand = _guard_candidates(resp, usage, "Gemini")
finish = str(cand.finish_reason.value if cand.finish_reason else "") or "UNKNOWN"
texts: list[str] = []
- calls: list[ToolCallPart] = []
for part in cand.content.parts if cand.content and cand.content.parts else []:
- if part.function_call is not None:
- fc = part.function_call
- # a genuine provider id is kept verbatim (and echoed back later); a synthetic
- # one is minted, marked by its prefix, only so the parts pair locally
- call_id = fc.id or _next_call_id()
- SIGNATURES.put(call_id, part.thought_signature)
- calls.append(
- ToolCallPart(id=call_id, name=fc.name or "", arguments=dict(fc.args or {}))
- )
- elif part.text and not part.thought:
+ if part.text and not part.thought:
texts.append(part.text)
text = "".join(texts)
- if not text and not calls:
+ if not text:
if finish in FATAL_FINISH:
raise ModelError(
f"Gemini produced no content (finish_reason={finish})", retryable=False, usage=usage
@@ -292,7 +191,7 @@ def extract_candidate(
)
raise ModelError(f"Gemini produced no content (finish_reason={finish})",
retryable=True, usage=usage)
- return text, calls, finish
+ return text, finish
# ===================================================================== gemini
@@ -585,7 +484,7 @@ class GeminiModel:
def generate(self, request: ChatRequest) -> ChatResponse:
contents = to_contents(request.messages)
warnings: list[str] = []
- state = {"config": self._config(request, warnings)}
+ state = {"config": self._config(request)}
def downgrade_thinking(err: ModelError) -> bool:
"""Free-retry hook: model rejects ThinkingConfig → retry without it, once."""
@@ -594,7 +493,7 @@ class GeminiModel:
with self._lock:
self._thinking_ok = False
warnings.append(f"{self.model} rejected thinking config; retrying without it: {err}")
- state["config"] = self._config(request, warnings)
+ state["config"] = self._config(request)
return True
# the caller's budget clips the retry deadline, never extends it
@@ -639,12 +538,11 @@ class GeminiModel:
resp.raw["wasted_usage"] = sum(wasted[1:], wasted[0])
return resp
- def _config(self, request: ChatRequest, warnings: list[str]) -> types.GenerateContentConfig:
+ def _config(self, request: ChatRequest) -> types.GenerateContentConfig:
return build_config(
request,
timeout_ms=int(self.timeout_s * 1000),
use_thinking=self._thinking_ok,
- warnings=warnings,
)
def _attempt_config(
@@ -722,10 +620,9 @@ class GeminiModel:
usage = parse_usage(resp, self.model) # BEFORE extract_candidate: its raises carry it
usage.latency_ms = sw.ms
usage.cost_usd = estimate_cost("gemini", self.model, usage)
- text, calls, finish = extract_candidate(resp, usage)
- usage.tool_calls = len(calls)
+ text, finish = extract_candidate(resp, usage)
parsed: Any = None
- if request.response_schema is not None and not calls:
+ if request.response_schema is not None:
try:
parsed = parse_json_lenient(text)
except JsonParseError as exc:
@@ -742,7 +639,7 @@ class GeminiModel:
retryable=True,
usage=usage,
) from exc
- elif finish in RETRYABLE_FINISH and not calls and not text.strip():
+ elif finish in RETRYABLE_FINISH and not text.strip():
raise ModelError(f"Gemini finish_reason={finish}", retryable=True, usage=usage)
raw: dict[str, Any] = {
"finish_reason": finish,
@@ -752,9 +649,7 @@ class GeminiModel:
}
if warnings:
raw["warnings"] = list(warnings)
- return ChatResponse(
- text=text, parsed=parsed, tool_calls=calls, finish_reason=finish, usage=usage, raw=raw
- )
+ return ChatResponse(text=text, parsed=parsed, finish_reason=finish, usage=usage, raw=raw)
# ===================================================================== gemini_image
diff --git a/harness/codeverse/models/openai.py b/harness/codeverse/models/openai.py
index 5fe8d72c..d81ed912 100644
--- a/harness/codeverse/models/openai.py
+++ b/harness/codeverse/models/openai.py
@@ -1,4 +1,4 @@
-"""ChatRequest ⇄ OpenAI Chat Completions dicts (messages, tools, response_format).
+"""ChatRequest ⇄ OpenAI Chat Completions dicts (messages, response_format).
Chat Completions (not Responses) was chosen because every OpenAI-compatible
endpoint (``OPENAI_BASE_URL``: vLLM, OpenRouter, Ollama, …) speaks it, while
@@ -7,7 +7,6 @@ the Responses API is OpenAI-only.
from __future__ import annotations
-import json
import logging
import threading
import time
@@ -20,9 +19,6 @@ from codeverse.contracts.chat import (
ChatResponse,
ImagePart,
TextPart,
- ToolCallPart,
- ToolResultPart,
- ToolSpec,
)
from codeverse.contracts.common import Usage
from codeverse.models.base import ModelError
@@ -76,9 +72,7 @@ def _image_block(p: ImagePart) -> dict[str, Any]:
def to_messages(messages: list[ChatMessage], system: str) -> list[dict[str, Any]]:
- """ChatMessages → Chat Completions messages. Tool results become ``role:
- tool`` messages; images inside tool results (unsupported there) are sent as
- a follow-up user message."""
+ """ChatMessages → Chat Completions messages."""
out: list[dict[str, Any]] = []
if system:
out.append({"role": "system", "content": system})
@@ -86,7 +80,6 @@ def to_messages(messages: list[ChatMessage], system: str) -> list[dict[str, Any]
if msg.role == "assistant":
out.append(_assistant_message(msg))
continue
- trailing_images: list[dict[str, Any]] = []
content: list[dict[str, Any]] = []
for p in msg.parts:
if isinstance(p, TextPart):
@@ -96,29 +89,8 @@ def to_messages(messages: list[ChatMessage], system: str) -> list[dict[str, Any]
if p.label:
content.append({"type": "text", "text": f"[image: {p.label}]"})
content.append(_image_block(p))
- elif isinstance(p, ToolResultPart):
- if content:
- out.append({"role": "user", "content": content})
- content = []
- out.append(
- {
- "role": "tool",
- "tool_call_id": p.call_id,
- "content": p.content or "(no output)",
- }
- )
- for img in p.images:
- if img.label:
- trailing_images.append(
- {"type": "text", "text": f"[image from {p.name}: {img.label}]"}
- )
- trailing_images.append(_image_block(img))
- elif isinstance(p, ToolCallPart):
- raise ModelError("ToolCallPart is only valid in assistant messages")
if content:
out.append({"role": "user", "content": content})
- if trailing_images:
- out.append({"role": "user", "content": trailing_images})
if not out or all(m["role"] == "system" for m in out):
raise ModelError("ChatRequest has no content to send")
return out
@@ -126,28 +98,7 @@ def to_messages(messages: list[ChatMessage], system: str) -> list[dict[str, Any]
def _assistant_message(msg: ChatMessage) -> dict[str, Any]:
text = "".join(p.text for p in msg.parts if isinstance(p, TextPart))
- calls = [
- {
- "id": p.id,
- "type": "function",
- "function": {"name": p.name, "arguments": json.dumps(p.arguments)},
- }
- for p in msg.parts
- if isinstance(p, ToolCallPart)
- ]
- m: dict[str, Any] = {"role": "assistant", "content": text or None}
- if calls:
- m["tool_calls"] = calls
- return m
-
-
-# --------------------------------------------------------------------- tools
-def to_tool(t: ToolSpec) -> dict[str, Any]:
- params = inline_refs(t.parameters) if t.parameters else {"type": "object", "properties": {}}
- return {
- "type": "function",
- "function": {"name": t.name, "description": t.description, "parameters": params},
- }
+ return {"role": "assistant", "content": text or None}
def response_format(schema: dict[str, Any], *, strict: bool) -> dict[str, Any]:
@@ -170,36 +121,19 @@ def build_kwargs(request: ChatRequest, model: str, *, strict_schema: bool) -> di
kw["reasoning_effort"] = effort
else:
kw["temperature"] = request.temperature
- if request.tools:
- kw["tools"] = [to_tool(t) for t in request.tools]
- kw["tool_choice"] = "auto"
if request.response_schema is not None:
kw["response_format"] = response_format(request.response_schema, strict=strict_schema)
return kw
# ------------------------------------------------------------------ response
-def parse_choice(choice: Any) -> tuple[str, list[ToolCallPart], str]:
- """→ ``(text, tool_calls, finish_reason)``; bad tool-call JSON → ModelError(retryable)."""
+def parse_choice(choice: Any) -> tuple[str, str]:
+ """→ ``(text, finish_reason)``."""
msg = choice.message
text = msg.content or ""
if getattr(msg, "refusal", None):
raise ModelError(f"OpenAI refusal: {msg.refusal}", retryable=False)
- calls: list[ToolCallPart] = []
- for tc in msg.tool_calls or []:
- fn = getattr(tc, "function", None)
- if fn is None:
- continue
- try:
- args = json.loads(fn.arguments or "{}")
- except json.JSONDecodeError as exc:
- raise ModelError(
- f"tool call {fn.name} has invalid JSON arguments: {exc}", retryable=True
- ) from exc
- if not isinstance(args, dict):
- args = {"value": args}
- calls.append(ToolCallPart(id=tc.id, name=fn.name, arguments=args))
- return text, calls, str(choice.finish_reason or "")
+ return text, str(choice.finish_reason or "")
# ===================================================================== openai
@@ -313,10 +247,10 @@ class OpenAIModel:
model=self.model, timeout=attempt_timeout_s(deadline, self.timeout_s), **kwargs)
if not completion.choices:
raise ModelError("OpenAI returned no choices", retryable=True)
- text, calls, finish = parse_choice(completion.choices[0])
- usage = self._usage(completion.usage, sw.ms, len(calls))
+ text, finish = parse_choice(completion.choices[0])
+ usage = self._usage(completion.usage, sw.ms)
parsed: Any = None
- if request.response_schema is not None and not calls:
+ if request.response_schema is not None:
try:
parsed = parse_json_lenient(text)
except JsonParseError as exc:
@@ -324,7 +258,7 @@ class OpenAIModel:
f"structured output is not valid JSON (finish_reason={finish}): {exc}",
retryable=finish != "length", usage=usage, # billed like a good reply
) from exc
- if not text and not calls:
+ if not text:
raise ModelError(
f"OpenAI returned no content (finish_reason={finish})",
retryable=finish != "content_filter", usage=usage,
@@ -334,15 +268,11 @@ class OpenAIModel:
"id": completion.id,
"model": completion.model,
}
- return ChatResponse(
- text=text, parsed=parsed, tool_calls=calls, finish_reason=finish, usage=usage, raw=raw
- )
+ return ChatResponse(text=text, parsed=parsed, finish_reason=finish, usage=usage, raw=raw)
- def _usage(self, u: Any, latency_ms: int, n_calls: int) -> Usage:
+ def _usage(self, u: Any, latency_ms: int) -> Usage:
if u is None:
- usage = Usage(
- backend="openai", model=self.model, tool_calls=n_calls, latency_ms=latency_ms
- )
+ usage = Usage(backend="openai", model=self.model, latency_ms=latency_ms)
else:
ptd = getattr(u, "prompt_tokens_details", None)
ctd = getattr(u, "completion_tokens_details", None)
@@ -357,7 +287,6 @@ class OpenAIModel:
output_tokens=max(0, completion - reasoning),
cached_tokens=cached,
thoughts_tokens=reasoning,
- tool_calls=n_calls,
latency_ms=latency_ms,
)
usage.cost_usd = estimate_cost("openai", self.model, usage)
diff --git a/harness/codeverse/models/parts.py b/harness/codeverse/models/parts.py
index 0f8ced7c..37f75abd 100644
--- a/harness/codeverse/models/parts.py
+++ b/harness/codeverse/models/parts.py
@@ -1,13 +1,10 @@
-"""Small helpers shared by the provider backends (images, timing, usage,
-bounded per-call-id cache)."""
+"""Small helpers shared by the provider backends (images, timing, usage)."""
from __future__ import annotations
import base64
import mimetypes
-import threading
import time
-from collections import OrderedDict
from pathlib import Path
from typing import Any
@@ -64,32 +61,6 @@ class Stopwatch:
self.ms = int((time.perf_counter() - self._t0) * 1000)
-class BoundedCache[T]:
- """Thread-safe bounded LRU ``str -> T`` map for per-call-id provider state
- (gemini thought signatures, anthropic thinking blocks). ``put`` skips falsy
- values; ``get`` returns ``None`` on a miss — a caller that needs a copy /
- empty default wraps it: ``list(cache.get(k) or [])``.
- """
-
- def __init__(self, capacity: int) -> None:
- self._d: OrderedDict[str, T] = OrderedDict()
- self._cap = capacity
- self._lock = threading.Lock()
-
- def put(self, key: str, value: T | None) -> None:
- if not value:
- return
- with self._lock:
- self._d[key] = value
- self._d.move_to_end(key)
- while len(self._d) > self._cap:
- self._d.popitem(last=False)
-
- def get(self, key: str) -> T | None:
- with self._lock:
- return self._d.get(key)
-
-
def classify_sdk_exception(exc: BaseException, sdk: Any, label: str,
*, extra_retry: frozenset[int] = frozenset()) -> Any:
"""The openai/anthropic SDK exception ladder → ``ModelError``.
diff --git a/harness/codeverse/models/retry.py b/harness/codeverse/models/retry.py
index 763f94d7..1ed962e4 100644
--- a/harness/codeverse/models/retry.py
+++ b/harness/codeverse/models/retry.py
@@ -49,7 +49,7 @@ from dataclasses import dataclass, field
from functools import partial
from typing import TYPE_CHECKING, Any, Literal
-from codeverse.contracts.chat import ChatRequest, ImagePart, TextPart, ToolResultPart
+from codeverse.contracts.chat import ChatRequest, ImagePart, TextPart
Outcome = Literal["ok", "429", "5xx", "error", "dead", "skip"]
@@ -1087,13 +1087,11 @@ DEFAULT_IMAGE_PX = 1024
def request_parts(request: ChatRequest) -> tuple[list[str], int]:
"""``(text blocks, number of image parts)`` of a request, system prompt and
- tool schemas included — everything the provider will count as prompt."""
+ response schema included — everything the provider will count as prompt."""
blocks: list[str] = []
images = 0
if request.system:
blocks.append(request.system)
- for tool in request.tools or ():
- blocks.append(f"{tool.name}{tool.description}{tool.parameters}")
if request.response_schema is not None:
blocks.append(str(request.response_schema))
for msg in request.messages:
@@ -1102,11 +1100,6 @@ def request_parts(request: ChatRequest) -> tuple[list[str], int]:
blocks.append(part.text)
elif isinstance(part, ImagePart):
images += 1
- elif isinstance(part, ToolResultPart):
- blocks.append(part.content)
- images += len(part.images)
- else: # ToolCallPart
- blocks.append(f"{part.name}{part.arguments}")
return blocks, images
diff --git a/harness/docs/INTERFACES.md b/harness/docs/INTERFACES.md
index a7c1da41..341bc0f8 100644
--- a/harness/docs/INTERFACES.md
+++ b/harness/docs/INTERFACES.md
@@ -62,7 +62,7 @@ resp = m.generate(ChatRequest(messages=[...], system=..., response_schema=..., t
# max_wait_s: the longest this ONE call may spend, retries included (None = models.retry.RETRY_DEADLINE_S = 900 s);
# GeminiModel clips its retry deadline to it; api_agent 20-120 s per turn, VlmJudge 240 s per sample, planner 300 s
# resp.raw["key"] = "…ab12" (the key that answered), resp.raw["attempts"] = round-trips issued (hedged siblings included)
-resp.parsed / resp.text / resp.tool_calls / resp.usage # Usage always has cost_usd (models.pricing)
+resp.parsed / resp.text / resp.usage # Usage always has cost_usd (models.pricing)
from codeverse.models.retry import KeyPool, KeyPoolExhausted
KeyPool(keys, *, rpm_per_key=900, tpm_per_key=None, cooldown_s=30, dead_cooldown_s=3600)
pool.acquire(*, tokens_hint=0, exclude=None, timeout_s=120) -> key # raises immediately when every key is dead/cooling past the deadline
@@ -88,11 +88,9 @@ Rules that callers must know:
only when *every* key fails the same way. A 429 while an untried key remains is
a **free** rotation (does not consume `max_attempts`); only when all keys are
throttled do 429s count against the budget with backoff.
-* **Δ** Gemini: `tools` + `response_schema` together → schema dropped, text parsed
- leniently into `.parsed`, warning in `resp.raw["warnings"]`. Gemini 3.x cannot
- disable thinking; keep `max_output_tokens ≥ ~1000` even for one-word answers.
+* Gemini 3.x cannot disable thinking; keep `max_output_tokens ≥ ~1000` even for
+ one-word answers.
* Anthropic 4.6+ models use adaptive thinking + `output_config.effort`.
- Tool-call ids must be echoed unchanged (they key provider caches).
* **Δ** `to_openai_strict_schema` never adds `null` to defaulted fields: a property
is nullable iff the *source* pydantic schema is (`T | None`); non-null defaults
are surfaced as a `[default: …]` description hint. Wire contract == pydantic contract.
diff --git a/harness/tests/models/test_anthropic.py b/harness/tests/models/test_anthropic.py
index 7499e591..fab32492 100644
--- a/harness/tests/models/test_anthropic.py
+++ b/harness/tests/models/test_anthropic.py
@@ -13,12 +13,8 @@ from codeverse.contracts.chat import (
ChatMessage,
ChatRequest,
ImagePart,
- ToolCallPart,
- ToolResultPart,
- ToolSpec,
)
from codeverse.models.anthropic import (
- THINKING_BLOCKS,
AnthropicModel,
build_kwargs,
classify_exception,
@@ -53,13 +49,6 @@ def tool_use(id_: str, name: str, inp: dict):
return NS(type="tool_use", id=id_, name=name, input=inp)
-class Thinking:
- type = "thinking"
-
- def model_dump(self):
- return {"type": "thinking", "thinking": "hmm", "signature": "sig"}
-
-
class FakeClient:
def __init__(self, script: list[Any]):
self.script, self.calls = script, []
@@ -154,7 +143,7 @@ def test_structured_output_via_forced_submit_tool():
r = m.generate(
ChatRequest(messages=[ChatMessage.user("x")], response_schema=schema, thinking="off")
)
- assert r.parsed == {"a": 1} and json.loads(r.text) == {"a": 1} and r.tool_calls == []
+ assert r.parsed == {"a": 1} and json.loads(r.text) == {"a": 1}
kw = fc.calls[0]
assert kw["tool_choice"] == {"type": "tool", "name": "submit"}
assert (
@@ -183,55 +172,6 @@ def test_output_config_json_mode():
assert fc.calls[0]["output_config"]["format"]["type"] == "json_schema"
-def test_tool_roundtrip_with_thinking_blocks_replayed():
- tool = ToolSpec(
- name="measure",
- description="d",
- parameters={"type": "object", "properties": {"part": {"type": "string"}}},
- )
- m, fc = make(
- [
- msg([Thinking(), tool_use("tu_9", "measure", {"part": "all"})], stop="tool_use"),
- msg([text("45 cm")]),
- ]
- )
- req = ChatRequest(messages=[ChatMessage.user("measure")], tools=[tool])
- r = m.generate(req)
- call = r.tool_calls[0]
- assert call.id == "tu_9" and call.arguments == {"part": "all"} and r.finish_reason == "tool_use"
- assert THINKING_BLOCKS.get("tu_9")
- msgs = [
- *req.messages,
- ChatMessage(role="assistant", parts=[call]),
- ChatMessage(
- role="tool",
- parts=[
- ToolResultPart(
- call_id="tu_9",
- name="measure",
- content="0.45",
- images=[ImagePart(data_b64=PNG_B64)],
- )
- ],
- ),
- ]
- r2 = m.generate(ChatRequest(messages=msgs, tools=[tool]))
- assert r2.text == "45 cm"
- sent = fc.calls[1]["messages"]
- assert (
- sent[1]["role"] == "assistant"
- and sent[1]["content"][0]["type"] == "thinking"
- and sent[1]["content"][1]["type"] == "tool_use"
- )
- tr = sent[2]["content"][0]
- assert (
- tr["type"] == "tool_result"
- and tr["tool_use_id"] == "tu_9"
- and tr["content"][1]["type"] == "image"
- )
- assert fc.calls[0]["tools"][0]["input_schema"]["properties"]["part"]["type"] == "string"
-
-
def test_images_base64_and_message_merging():
msgs = to_messages(
[
@@ -292,10 +232,8 @@ def test_missing_key_is_loud(monkeypatch):
get_settings.cache_clear()
-def test_tool_call_part_in_user_message_ignored_but_assistant_first_gets_user_prefix():
- msgs = to_messages(
- [ChatMessage(role="assistant", parts=[ToolCallPart(id="t", name="n", arguments={})])]
- )
+def test_assistant_first_gets_user_prefix():
+ msgs = to_messages([ChatMessage.assistant("hi")])
assert msgs[0]["role"] == "user" and msgs[1]["role"] == "assistant"
diff --git a/harness/tests/models/test_gemini.py b/harness/tests/models/test_gemini.py
index 088dc15f..5c753f51 100644
--- a/harness/tests/models/test_gemini.py
+++ b/harness/tests/models/test_gemini.py
@@ -15,13 +15,10 @@ from codeverse.contracts.chat import (
ChatRequest,
ImagePart,
TextPart,
- ToolResultPart,
- ToolSpec,
)
from codeverse.contracts.plan import StaticPlan
from codeverse.models.base import ModelError
from codeverse.models.gemini import (
- SIGNATURES,
GeminiModel,
build_config,
classify_exception,
@@ -34,14 +31,6 @@ PNG_1PX = base64.b64decode(
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=="
)
-#: the one tool spec every tool-call test uses.
-MEASURE_TOOL = ToolSpec(
- name="measure",
- description="m",
- parameters={"type": "object", "properties": {"part": {"type": "string"}}},
-)
-
-
def text_response(
text: str, *, finish: str = "STOP", thoughts: int = 3
) -> types.GenerateContentResponse:
@@ -62,21 +51,6 @@ def text_response(
)
-def call_response(
- name: str, args: dict, sig: bytes | None = b"sig-bytes"
-) -> types.GenerateContentResponse:
- part = types.Part(function_call=types.FunctionCall(name=name, args=args))
- part.thought_signature = sig
- return types.GenerateContentResponse(
- candidates=[
- types.Candidate(content=types.Content(role="model", parts=[part]), finish_reason="STOP")
- ],
- usage_metadata=types.GenerateContentResponseUsageMetadata(
- prompt_token_count=10, candidates_token_count=5
- ),
- )
-
-
class FakeModels:
def __init__(self, script: list[Any], key: str, log: list[dict]):
self.script, self.key, self.log = script, key, log
@@ -266,55 +240,6 @@ def test_thinking_rejected_falls_back_without_thinking_config():
assert log[0]["config"].thinking_config is None
-def test_tool_call_roundtrip_and_signature_cache():
- tool = MEASURE_TOOL
- m, log, _ = make_model([call_response("measure", {"part": "all"}), text_response("45 cm")])
- req = ChatRequest(messages=[ChatMessage.user("measure it")], tools=[tool])
- r = m.generate(req)
- assert (
- r.tool_calls
- and r.tool_calls[0].name == "measure"
- and r.tool_calls[0].arguments == {"part": "all"}
- )
- assert r.usage.tool_calls == 1 and r.text == ""
- cfg = log[0]["config"]
- assert cfg.tools[0].function_declarations[0].name == "measure"
- assert cfg.response_mime_type is None
- call = r.tool_calls[0]
- assert SIGNATURES.get(call.id) == b"sig-bytes"
- msgs = [
- *req.messages,
- ChatMessage(role="assistant", parts=[call]),
- ChatMessage(
- role="tool",
- parts=[
- ToolResultPart(call_id=call.id, name="measure", content=json.dumps({"h": 0.45}))
- ],
- ),
- ]
- r2 = m.generate(ChatRequest(messages=msgs, tools=[tool]))
- assert r2.text == "45 cm" and not r2.tool_calls
- contents = log[1]["contents"]
- assert [c.role for c in contents] == ["user", "model", "user"]
- fc_part = contents[1].parts[0]
- assert fc_part.function_call.name == "measure" and fc_part.thought_signature == b"sig-bytes"
- fr = contents[2].parts[0].function_response
- assert fr.name == "measure" and fr.response == {"h": 0.45}
-
-
-def test_tools_plus_schema_warns_and_drops_json_mode():
- tool = ToolSpec(name="t", description="d", parameters={"type": "object", "properties": {}})
- m, log, _ = make_model([text_response('{"a": 1}')])
- r = m.generate(
- ChatRequest(
- messages=[ChatMessage.user("x")], tools=[tool], response_schema={"type": "object"}
- )
- )
- assert r.parsed == {"a": 1}
- assert log[0]["config"].response_mime_type is None
- assert any("response_schema ignored" in w for w in r.raw["warnings"])
-
-
def test_images_inline_in_order(tmp_path):
p = tmp_path / "img.png"
p.write_bytes(PNG_1PX)
@@ -425,85 +350,6 @@ def test_a_charged_but_invalid_reply_carries_its_usage_on_the_error():
assert ei.value.usage.cost_usd > 0
-def _resp_with_calls(parts: list[types.Part]) -> types.GenerateContentResponse:
- return types.GenerateContentResponse(
- candidates=[
- types.Candidate(content=types.Content(role="model", parts=parts), finish_reason="STOP")
- ],
- usage_metadata=types.GenerateContentResponseUsageMetadata(
- prompt_token_count=10, candidates_token_count=5
- ),
- )
-
-
-def test_synthetic_call_ids_are_never_echoed_to_the_provider():
- """Locally synthesized tool ids never travel back to Gemini."""
- from codeverse.models.gemini import is_synthetic_call_id
-
- tool = ToolSpec(
- name="measure", description="m", parameters={"type": "object", "properties": {}}
- )
- m, log, _ = make_model([call_response("measure", {"part": "all"}), text_response("ok")])
- req = ChatRequest(messages=[ChatMessage.user("x")], tools=[tool])
- r = m.generate(req)
- call = r.tool_calls[0]
- assert is_synthetic_call_id(call.id)
- msgs = [
- *req.messages,
- ChatMessage(role="assistant", parts=[call]),
- ChatMessage(
- role="tool", parts=[ToolResultPart(call_id=call.id, name="measure", content="{}")]
- ),
- ]
- m.generate(ChatRequest(messages=msgs, tools=[tool]))
- contents = log[1]["contents"]
- assert contents[1].parts[0].function_call.id is None, "an invented id must not reach Gemini"
- assert contents[2].parts[0].function_response.id is None
-
-
-def test_two_parallel_same_name_calls_keep_their_provider_ids():
- """Parallel same-name calls preserve provider ids and signatures."""
- from codeverse.models.gemini import is_synthetic_call_id
-
- parts = []
- for cid, target in (("id-a", "seat"), ("id-b", "leg")):
- p = types.Part(
- function_call=types.FunctionCall(name="measure", args={"part": target}, id=cid)
- )
- p.thought_signature = f"sig-{cid}".encode()
- parts.append(p)
- tool = MEASURE_TOOL
- m, log, _ = make_model([_resp_with_calls(parts), text_response("done")])
- req = ChatRequest(messages=[ChatMessage.user("measure both")], tools=[tool])
- r = m.generate(req)
- a, b = r.tool_calls
- assert (a.id, b.id) == ("id-a", "id-b") and a.name == b.name == "measure"
- assert not is_synthetic_call_id(a.id), "a provider id must not be mistaken for a minted one"
- assert SIGNATURES.get("id-a") == b"sig-id-a" and SIGNATURES.get("id-b") == b"sig-id-b"
- msgs = [
- *req.messages,
- ChatMessage(role="assistant", parts=[a, b]),
- ChatMessage(
- role="tool",
- parts=[ # results deliberately out of order: the ids do the pairing
- ToolResultPart(call_id=b.id, name="measure", content=json.dumps({"part": "leg"})),
- ToolResultPart(call_id=a.id, name="measure", content=json.dumps({"part": "seat"})),
- ],
- ),
- ]
- m.generate(ChatRequest(messages=msgs, tools=[tool]))
- model_parts = log[1]["contents"][1].parts
- assert [fp.function_call.id for fp in model_parts] == ["id-a", "id-b"]
- assert [fp.thought_signature for fp in model_parts] == [b"sig-id-a", b"sig-id-b"]
- responses = log[1]["contents"][2].parts
- assert [
- (fp.function_response.id, fp.function_response.response["part"]) for fp in responses
- ] == [
- ("id-b", "leg"),
- ("id-a", "seat"),
- ]
-
-
def test_max_output_tokens_eaten_by_thinking_carries_the_thought_tokens():
"""A MAX_TOKENS failure reports the thinking tokens it consumed."""
resp = types.GenerateContentResponse(
@@ -565,12 +411,10 @@ def test_ipv4_transport_toggle(monkeypatch):
assert _ipv4_client_args() == {}
-def test_merge_stream_chunks_carries_tool_calls_and_signatures():
- """Stream merging preserves ordered tool calls and thought signatures."""
+def test_merge_stream_chunks_concatenates_text_and_keeps_final_usage():
+ """Stream merging concatenates text parts in order; usage comes from the final chunk."""
from codeverse.models.gemini import _merge_stream_chunks, extract_candidate
- fc = types.Part(function_call=types.FunctionCall(name="measure", args={"part": "Hull"}))
- fc.thought_signature = b"sig-bytes"
c1 = types.GenerateContentResponse(
candidates=[
types.Candidate(
@@ -581,7 +425,7 @@ def test_merge_stream_chunks_carries_tool_calls_and_signatures():
c2 = types.GenerateContentResponse(
candidates=[
types.Candidate(
- content=types.Content(role="model", parts=[types.Part.from_text(text="lo"), fc]),
+ content=types.Content(role="model", parts=[types.Part.from_text(text="lo")]),
finish_reason="STOP",
)
],
@@ -590,8 +434,6 @@ def test_merge_stream_chunks_carries_tool_calls_and_signatures():
),
)
merged = _merge_stream_chunks([c1, c2])
- text, calls, finish = extract_candidate(merged)
+ text, finish = extract_candidate(merged)
assert text == "hello" and finish == "STOP"
assert merged.usage_metadata.prompt_token_count == 9
- assert len(calls) == 1 and calls[0].name == "measure" and calls[0].arguments == {"part": "Hull"}
- assert SIGNATURES.get(calls[0].id) == b"sig-bytes"
diff --git a/harness/tests/models/test_live_gemini.py b/harness/tests/models/test_live_gemini.py
index 56a97455..430204a8 100644
--- a/harness/tests/models/test_live_gemini.py
+++ b/harness/tests/models/test_live_gemini.py
@@ -2,17 +2,12 @@
from __future__ import annotations
-import json
-
import pytest
from codeverse.contracts.chat import (
ChatMessage,
ChatRequest,
ImagePart,
- TextPart,
- ToolResultPart,
- ToolSpec,
)
from codeverse.contracts.plan import StaticPlan
from codeverse.models import get_chat_model
@@ -85,43 +80,3 @@ def test_vision(model, tmp_path):
)
)
assert "red" in r.text.lower()
-
-
-def test_tool_roundtrip(model):
- tool = ToolSpec(
- name="measure",
- description="Measure the bounding box (meters) of the built object; returns extents x,y,z.",
- parameters={
- "type": "object",
- "properties": {"part": {"type": "string", "description": "part name or 'all'"}},
- "required": ["part"],
- },
- )
- msgs = [
- ChatMessage.user(
- "Use the measure tool on the whole object, then report its height in cm. Never guess."
- )
- ]
- r = model.generate(
- ChatRequest(messages=msgs, tools=[tool], thinking="low", max_output_tokens=2000)
- )
- assert r.tool_calls and r.tool_calls[0].name == "measure"
- call = r.tool_calls[0]
- parts = ([TextPart(text=r.text)] if r.text else []) + [call]
- msgs.append(ChatMessage(role="assistant", parts=parts))
- msgs.append(
- ChatMessage(
- role="tool",
- parts=[
- ToolResultPart(
- call_id=call.id,
- name="measure",
- content=json.dumps({"extents_m": [0.35, 0.35, 0.45]}),
- )
- ],
- )
- )
- r2 = model.generate(
- ChatRequest(messages=msgs, tools=[tool], thinking="low", max_output_tokens=2000)
- )
- assert "45" in r2.text and not r2.tool_calls
diff --git a/harness/tests/models/test_openai.py b/harness/tests/models/test_openai.py
index bbbfcd9c..560b8cb3 100644
--- a/harness/tests/models/test_openai.py
+++ b/harness/tests/models/test_openai.py
@@ -2,14 +2,13 @@
from __future__ import annotations
-import json
from types import SimpleNamespace as NS
from typing import Any
import httpx
import pytest
-from codeverse.contracts.chat import ChatMessage, ChatRequest, ImagePart, ToolResultPart, ToolSpec
+from codeverse.contracts.chat import ChatMessage, ChatRequest, ImagePart
from codeverse.models.base import ModelError
from codeverse.models.openai import (
OpenAIModel,
@@ -25,7 +24,6 @@ PNG_B64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwACh
def completion(
content: str | None,
*,
- tool_calls=None,
finish="stop",
prompt=100,
comp=30,
@@ -37,7 +35,7 @@ def completion(
model="gpt-5.6-sol",
choices=[
NS(
- message=NS(content=content, tool_calls=tool_calls, refusal=None),
+ message=NS(content=content, refusal=None),
finish_reason=finish,
)
],
@@ -50,10 +48,6 @@ def completion(
)
-def tc(id_, name, args: dict):
- return NS(id=id_, type="function", function=NS(name=name, arguments=json.dumps(args)))
-
-
class FakeClient:
def __init__(self, script: list[Any]):
self.script, self.calls = script, []
@@ -163,62 +157,6 @@ def test_bad_json_retried():
assert r.parsed == {"ok": 1} and len(fc.calls) == 2
-def test_tool_roundtrip_and_images_in_tool_results():
- tool = ToolSpec(
- name="measure",
- description="d",
- parameters={"type": "object", "properties": {"part": {"type": "string"}}},
- )
- bad = NS(id="bad", type="function", function=NS(name="measure", arguments="{not json"))
- m, fc = make(
- [
- completion(None, tool_calls=[bad], finish="tool_calls"),
- completion(
- None, tool_calls=[tc("call_1", "measure", {"part": "all"})], finish="tool_calls"
- ),
- completion("45 cm"),
- ]
- )
- req = ChatRequest(messages=[ChatMessage.user("measure")], tools=[tool])
- r = m.generate(req)
- call = r.tool_calls[0]
- assert (
- call.id == "call_1"
- and call.arguments == {"part": "all"}
- and r.finish_reason == "tool_calls"
- and r.text == ""
- )
- assert (
- fc.calls[1]["tools"][0]["function"]["name"] == "measure"
- and fc.calls[1]["tool_choice"] == "auto"
- )
- msgs = [
- *req.messages,
- ChatMessage(role="assistant", parts=[call]),
- ChatMessage(
- role="tool",
- parts=[
- ToolResultPart(
- call_id="call_1",
- name="measure",
- content="0.45",
- images=[ImagePart(data_b64=PNG_B64)],
- )
- ],
- ),
- ]
- r2 = m.generate(ChatRequest(messages=msgs, tools=[tool]))
- assert r2.text == "45 cm"
- sent = fc.calls[2]["messages"]
- assert (
- sent[1]["role"] == "assistant"
- and sent[1]["tool_calls"][0]["function"]["arguments"] == '{"part": "all"}'
- )
- assert sent[2] == {"role": "tool", "tool_call_id": "call_1", "content": "0.45"}
- assert sent[3]["role"] == "user" and sent[3]["content"][0]["type"] == "image_url"
- assert sent[3]["content"][0]["image_url"]["url"].startswith("data:image/png;base64,")
-
-
def test_images_data_urls():
msgs = to_messages(
[