Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(rag): align tool selector contract with execution (#1928)

XZH43ad667a35fa
diff --git a/deeptutor/core/tool_protocol.py b/deeptutor/core/tool_protocol.py
index edbfd8dc1..1ca559339 100644
--- a/deeptutor/core/tool_protocol.py
+++ b/deeptutor/core/tool_protocol.py
@@ -34,6 +34,17 @@ class ToolParameter:
default: Any = None
enum: list[str] | None = None
items: dict[str, Any] | None = None
+ #: Emit ``"type": [<type>, "null"]`` so a strictly-required property still
+ #: has a genuine absence value. Endpoints that normalise optional
+ #: properties into required fields reject ``null`` otherwise, and the model
+ #: falls back to placeholder values (issue #1928).
+ nullable: bool = False
+ #: Numeric bounds and array length bounds are emitted verbatim so the
+ #: advertised contract matches what execution enforces.
+ minimum: float | None = None
+ maximum: float | None = None
+ min_items: int | None = None
+ max_items: int | None = None
#: Keep this argument's value out of the ``tool_call`` event the trace
#: renders. The model still sends it and the tool still receives it — this
#: only stops it from being shown back to the person the tool is acting
@@ -44,9 +55,20 @@ class ToolParameter:
def to_schema(self) -> dict[str, Any]:
"""Convert to JSON Schema property dict."""
- schema: dict[str, Any] = {"type": self.type, "description": self.description}
+ schema: dict[str, Any] = {
+ "type": [self.type, "null"] if self.nullable else self.type,
+ "description": self.description,
+ }
if self.enum:
schema["enum"] = self.enum
+ if self.minimum is not None:
+ schema["minimum"] = self.minimum
+ if self.maximum is not None:
+ schema["maximum"] = self.maximum
+ if self.min_items is not None:
+ schema["minItems"] = self.min_items
+ if self.max_items is not None:
+ schema["maxItems"] = self.max_items
if self.type == "array":
schema["items"] = self.items if self.items is not None else {"type": "string"}
return schema
diff --git a/deeptutor/tools/builtin/__init__.py b/deeptutor/tools/builtin/__init__.py
index 94b96676b..4e113bffe 100644
--- a/deeptutor/tools/builtin/__init__.py
+++ b/deeptutor/tools/builtin/__init__.py
@@ -5,6 +5,7 @@ from __future__ import annotations
import asyncio
import json
import logging
+import math
from typing import Any
from deeptutor.core.tool_protocol import BaseTool, ToolDefinition, ToolParameter, ToolResult
@@ -102,6 +103,61 @@ def _rag_sources(result: dict[str, Any], *, query: str, kb_name: str) -> list[di
return sources
+def _rag_selector_error(message: str) -> ToolResult:
+ return ToolResult(
+ content=message,
+ success=False,
+ metadata={"evidence_kind": "exact_source_visual", "error_type": "invalid_selector"},
+ )
+
+
+def _normalize_rag_selectors(kwargs: dict[str, Any]) -> dict[str, Any] | str:
+ """Validate the optional exact-source selectors at the LLM boundary.
+
+ Returns the normalized selector map, or a correctable error string.
+ A legacy ``region=[]`` placeholder counts as omission (full page); every
+ other malformed selector is rejected visibly — never clamped, dropped,
+ or rerouted to replacement evidence (issue #1928).
+ """
+ selectors = {
+ key: kwargs.get(key)
+ for key in ("source_path", "asset_id", "figure", "page", "region", "source_hash")
+ }
+ page = selectors["page"]
+ if page is not None:
+ if isinstance(page, bool) or not isinstance(page, int):
+ return (
+ f"Invalid page {page!r}: PDF pages are one-based integers. "
+ "Resend with an integer page >= 1, or null to omit the page."
+ )
+ if page < 1:
+ return (
+ f"Invalid page {page}: PDF pages are one-based. "
+ "Resend with a page >= 1, or null to omit the page."
+ )
+ region = selectors["region"]
+ if region is not None:
+ if isinstance(region, list) and not region:
+ selectors["region"] = None
+ elif not (
+ isinstance(region, list)
+ and len(region) == 4
+ and all(
+ not isinstance(x, bool)
+ and isinstance(x, (int, float))
+ and math.isfinite(x)
+ and 0 <= x <= 1
+ for x in region
+ )
+ ):
+ return (
+ f"Invalid region {region!r}: a crop must be four finite numbers "
+ "[x0, y0, x1, y1] within 0..1. Resend with a valid crop, or "
+ "null to inspect the full page."
+ )
+ return selectors
+
+
class RAGTool(_PromptHintsMixin, BaseTool):
def get_definition(self) -> ToolDefinition:
return ToolDefinition(
@@ -122,37 +178,46 @@ class RAGTool(_PromptHintsMixin, BaseTool):
name="source_path",
type="string",
required=False,
+ nullable=True,
description="Exact document path from kb_files for visual/page inspection; prevents substitution across sources.",
),
ToolParameter(
name="asset_id",
type="string",
required=False,
+ nullable=True,
description="Exact immutable source visual ID to inspect again on a follow-up.",
),
ToolParameter(
name="figure",
type="string",
required=False,
+ nullable=True,
description="Exact figure/table label, e.g. Figure 3.2 or 图3.2. Ambiguous references require source/page selection.",
),
ToolParameter(
name="page",
type="integer",
required=False,
+ nullable=True,
+ minimum=1,
description="One-based PDF page for vector diagrams, sparse captions, tables or missing crops. Requires source_path.",
),
ToolParameter(
name="region",
type="array",
- items={"type": "number"},
+ items={"type": "number", "minimum": 0, "maximum": 1},
required=False,
- description="Optional PDF crop [x0,y0,x1,y1] normalized to 0..1; retains original layout at higher detail.",
+ nullable=True,
+ min_items=4,
+ max_items=4,
+ description="Optional PDF crop [x0,y0,x1,y1] normalized to 0..1; retains original layout at higher detail. Omitted or null means the full page.",
),
ToolParameter(
name="source_hash",
type="string",
required=False,
+ nullable=True,
description="Source document ID from an earlier citation; rejects changed source-page references.",
),
],
@@ -172,8 +237,11 @@ class RAGTool(_PromptHintsMixin, BaseTool):
from deeptutor.services.rag.source_visuals import figure_labels, retrieve_visual
exact_keys = ("source_path", "asset_id", "figure", "page", "region", "source_hash")
+ selectors = _normalize_rag_selectors(kwargs)
+ if isinstance(selectors, str):
+ return _rag_selector_error(selectors)
if any(
- kwargs.get(key) is not None and kwargs.get(key) != "" for key in exact_keys
+ selectors.get(key) is not None and selectors.get(key) != "" for key in exact_keys
) or figure_labels(query):
from deeptutor.multi_user.knowledge_access import resolve_for_rag
from deeptutor.services.rag.kb_paths import resolve_kb_dir
@@ -189,7 +257,7 @@ class RAGTool(_PromptHintsMixin, BaseTool):
resolve_kb_dir(resource.base_dir, resource.name),
kb_name,
query=query,
- **{key: kwargs[key] for key in exact_keys if kwargs.get(key) is not None},
+ **{key: selectors[key] for key in exact_keys if selectors.get(key) is not None},
)
except (OSError, ValueError, TypeError) as exc:
return ToolResult(content=f"Source visual inspection failed: {exc}", success=False)
@@ -206,10 +274,12 @@ class RAGTool(_PromptHintsMixin, BaseTool):
},
model_message=evidence.model_message() if vision_supported else None,
)
+ normalized_away = {key for key in exact_keys if selectors.get(key) is None}
extra_kwargs = {
key: value
for key, value in kwargs.items()
if key not in {"query", "kb_name", "event_sink", "_vision_supported"}
+ and key not in normalized_away
}
result = await rag_search(
diff --git a/tests/tools/test_rag_selector_contract.py b/tests/tools/test_rag_selector_contract.py
new file mode 100644
index 000000000..5407e9a5c
--- /dev/null
+++ b/tests/tools/test_rag_selector_contract.py
@@ -0,0 +1,159 @@
+"""RAG tool selector contract regressions (issue #1928).
+
+Offline, LLM-free checks on a synthetic single-page PDF: the advertised
+schema must match what execution enforces (optional selectors accept null,
+pages are one-based integers, crops are four coordinates in 0..1), legacy
+``region=[]`` placeholders mean "full page" instead of switching retrieval
+modes, and malformed selectors fail visibly with a correctable message
+instead of being clamped or replaced with other evidence.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from types import SimpleNamespace
+
+import pymupdf
+import pytest
+
+from deeptutor.services.llm.provider_core.openai_responses.converters import convert_tools
+from deeptutor.tools.builtin import RAGTool
+
+
+def kb(tmp_path: Path) -> Path:
+ raw = tmp_path / "kb" / "raw"
+ raw.mkdir(parents=True)
+ pdf = pymupdf.open()
+ page = pdf.new_page()
+ page.insert_text((50, 50), "Manual: inspect the pump seal before restart")
+ page.draw_line((50, 90), (300, 90))
+ pdf.save(raw / "manual.pdf", deflate=True)
+ pdf.close()
+ return raw.parent
+
+
+def rag_calls(kb: Path, monkeypatch: pytest.MonkeyPatch):
+ """Record semantic retrieval; visual resolution is patched per test."""
+ calls: dict[str, list] = {"rag_search": [], "resolve_for_rag": []}
+
+ async def fake_rag_search(**kwargs):
+ calls["rag_search"].append(kwargs)
+ return {"answer": "semantic answer", "sources": []}
+
+ def fake_resolve(kb_name):
+ calls["resolve_for_rag"].append(kb_name)
+ return SimpleNamespace(base_dir=kb.parent, name=kb.name)
+
+ monkeypatch.setattr("deeptutor.multi_user.knowledge_access.resolve_for_rag", fake_resolve)
+ monkeypatch.setattr("deeptutor.tools.rag_tool.rag_search", fake_rag_search)
+ return calls
+
+
+def _schema() -> dict:
+ schema = RAGTool().get_definition().to_openai_schema()["function"]["parameters"]
+ assert isinstance(schema, dict)
+ return schema
+
+
+def test_advertised_schema_matches_execution_contract():
+ schema = _schema()
+ properties = schema["properties"]
+ assert schema["required"] == ["query", "kb_name"]
+ assert properties["query"]["type"] == "string"
+ assert properties["kb_name"]["type"] == "string"
+ for name in ("source_path", "asset_id", "figure", "source_hash"):
+ assert properties[name]["type"] == ["string", "null"], name
+ assert properties["page"]["type"] == ["integer", "null"]
+ assert properties["page"]["minimum"] == 1
+ region = properties["region"]
+ assert region["type"] == ["array", "null"]
+ assert region["minItems"] == 4 and region["maxItems"] == 4
+ assert region["items"] == {"type": "number", "minimum": 0, "maximum": 1}
+ assert "full page" in region["description"]
+
+
+def test_responses_conversion_retains_selector_semantics():
+ converted = convert_tools([RAGTool().get_definition().to_openai_schema()])
+ assert len(converted) == 1
+ parameters = converted[0]["parameters"]
+ assert parameters["properties"]["page"]["minimum"] == 1
+ assert parameters["properties"]["page"]["type"] == ["integer", "null"]
+ region = parameters["properties"]["region"]
+ assert region["minItems"] == 4 and region["maxItems"] == 4
+ assert region["items"] == {"type": "number", "minimum": 0, "maximum": 1}
+ assert parameters["properties"]["source_hash"]["type"] == ["string", "null"]
+
+
+async def test_null_and_omitted_selectors_stay_semantic(kb, rag_calls):
+ for extra in (
+ {"region": []},
+ {"region": None, "page": None, "source_path": None},
+ {},
+ ):
+ result = await RAGTool().execute(query="pump seal", kb_name=kb.name, **extra)
+ assert result.success, extra
+ assert result.content == "semantic answer"
+ assert len(rag_calls["rag_search"]) == 3
+ assert rag_calls["resolve_for_rag"] == []
+
+
+async def test_page_with_legacy_empty_crop_returns_full_page(kb, rag_calls):
+ result = await RAGTool().execute(
+ query="Inspect this source",
+ kb_name=kb.name,
+ source_path="manual.pdf",
+ page=1,
+ region=[],
+ _vision_supported=True,
+ )
+ assert result.success
+ assert result.model_message["content"][1]["type"] == "image_url"
+ assert "pump seal" in result.content
+ assert "region=" not in result.sources[0]["visual_asset_url"]
+ assert rag_calls["rag_search"] == []
+
+
+async def test_valid_crop_returns_intended_region_with_provenance(kb, rag_calls):
+ result = await RAGTool().execute(
+ query="Inspect this source",
+ kb_name=kb.name,
+ source_path="manual.pdf",
+ page=1,
+ region=[0, 0, 0.6, 0.3],
+ _vision_supported=True,
+ )
+ assert result.success
+ assert "region=0%2C0%2C0.6%2C0.3" in result.sources[0]["visual_asset_url"]
+ assert result.sources[0]["source_document_id"]
+ assert rag_calls["rag_search"] == []
+
+
+ ("extra", "fragment"),
+ [
+ ({"page": 0, "region": []}, "one-based"),
+ ({"page": -2}, "one-based"),
+ ({"page": 1.5, "source_path": "manual.pdf"}, "integer"),
+ ({"page": "1", "source_path": "manual.pdf"}, "integer"),
+ ({"source_path": "manual.pdf", "page": 1, "region": [0, 1]}, "four finite numbers"),
+ ({"source_path": "manual.pdf", "page": 1, "region": [0, 0, 2, 1]}, "0..1"),
+ ({"source_path": "manual.pdf", "page": 1, "region": [0, 0, 0.5]}, "four finite numbers"),
+ ({"source_path": "manual.pdf", "page": 1, "region": "0,0,1,1"}, "four finite numbers"),
+ ],
+)
+async def test_malformed_selectors_fail_visibly_without_substitution(
+ kb, rag_calls, extra, fragment
+):
+ result = await RAGTool().execute(query="Inspect this source", kb_name=kb.name, **extra)
+ assert not result.success
+ assert fragment in result.content
+ assert result.sources == []
+ assert result.metadata["error_type"] == "invalid_selector"
+ assert rag_calls["rag_search"] == []
+ assert rag_calls["resolve_for_rag"] == []