RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(llm): drop Claude sampling params rejected by Bedrock and Anthropic (#19295)
VincentLambert2b8861ceb899
diff --git a/rag/llm/chat_model.py b/rag/llm/chat_model.pyindex 5b5870c6ce..d9ae2716bb 100644--- a/rag/llm/chat_model.py+++ b/rag/llm/chat_model.py@@ -112,6 +112,68 @@ LITELLM_ALLOWED_GEN_CONF_KEYS = ALLOWED_GEN_CONF_KEYS | frozenset(})+# Claude models that reject every sampling parameter (temperature / top_p / top_k -> HTTP 400).+_CLAUDE_NO_SAMPLING_MARKERS = (+ "claude-opus-4-7",+ "claude-opus-4-8",+ "claude-opus-5",+ "claude-sonnet-5",+ "claude-fable",+ "claude-mythos",+)++# First Claude generation that rejects ``temperature`` and ``top_p`` being set together, on the+# Anthropic API itself as well as through Bedrock (Anthropic API release notes, 2025-08-05:+# "Opus 4.1 does not allow both temperature and top_p parameters to be specified"; Haiku 4.5+# migration guide: "Use only temperature OR top_p, not both. Setting both returns a 400 error").+# Claude 3.x and Claude 4.0 (Opus 4 / Sonnet 4) still accept the pair.+_CLAUDE_TEMPERATURE_XOR_TOP_P_SINCE = (4, 1)++# ``claude-sonnet-4-5[-20250929]``, ``eu.anthropic.claude-opus-4-1-20250805-v1:0``, ``claude-fable-5-1``.+# The minor version is at most two digits so a date suffix (``claude-sonnet-4-20250514``) is not read as one.+_CLAUDE_VERSION_RE = re.compile(r"claude-(?:opus|sonnet|haiku|fable|mythos)-(\d+)(?:-(\d{1,2}))?(?!\d)")+# Legacy naming: ``claude-3-5-sonnet-20241022``, ``anthropic.claude-3-7-sonnet-20250219-v1:0``, ``claude-3-haiku``.+_CLAUDE_LEGACY_VERSION_RE = re.compile(r"claude-(\d)(?:-(\d))?-(?:opus|sonnet|haiku)")+++def _claude_version(model_name_lower: str) -> tuple[int, int] | None:+ """Return the ``(major, minor)`` Claude generation parsed from a model name, or ``None`` when unknown."""+ match = _CLAUDE_VERSION_RE.search(model_name_lower) or _CLAUDE_LEGACY_VERSION_RE.search(model_name_lower)+ if not match:+ return None+ return int(match.group(1)), int(match.group(2) or 0)+++def _apply_claude_sampling_policy(model_name_lower: str, *targets: dict) -> None:+ """Drop, in place on every ``targets`` dict, the sampling parameters a Claude model would reject.++ The rules are properties of the model generation, not of the gateway, so they apply to Claude+ served by Anthropic directly *and* through Bedrock (``eu.anthropic.claude-sonnet-4-6``):++ * Opus 4.7+, Sonnet 5, Fable/Mythos: no sampling parameter is accepted at all.+ * Claude 4.1 and later: ``temperature`` and ``top_p`` cannot both be specified (HTTP 400,+ "temperature and top_p cannot both be specified for this model"). ``temperature`` is the+ primary UI knob, so ``top_p`` is the one dropped.+ * Claude 3.x and 4.0 accept the pair and are left untouched. A Claude name whose generation+ cannot be parsed is treated as recent, since a dropped ``top_p`` beats a failing chat.++ Every drop is logged at WARNING level so the user can find out why a setting is not honoured.+ """+ if "claude" not in model_name_lower:+ return+ if any(marker in model_name_lower for marker in _CLAUDE_NO_SAMPLING_MARKERS):+ removed = [key for target in targets for key in ("temperature", "top_p", "top_k") if target.pop(key, None) is not None]+ if removed:+ logging.warning("Claude sampling policy: dropped %s for model %s (no sampling parameter accepted)", "/".join(sorted(set(removed))), model_name_lower)+ return+ version = _claude_version(model_name_lower)+ if version is not None and version < _CLAUDE_TEMPERATURE_XOR_TOP_P_SINCE:+ return+ if any("temperature" in target for target in targets) and any("top_p" in target for target in targets):+ for target in targets:+ target.pop("top_p", None)+ logging.warning("Claude sampling policy: dropped top_p for model %s (temperature and top_p cannot both be specified)", model_name_lower)+def _apply_model_family_policies(model_name: str,@@ -204,10 +266,8 @@ def _apply_model_family_policies(for key in ("temperature", "top_p", "logprobs", "top_logprobs"):sanitized_gen_conf.pop(key, None)sanitized_kwargs.pop(key, None)- elif provider == SupportedLiteLLMProvider.Anthropic and model_name_lower in {"claude-opus-4-7", "claude-opus-4-8"}:- for key in ("temperature", "top_p", "top_k"):- sanitized_gen_conf.pop(key, None)- sanitized_kwargs.pop(key, None)+ elif provider in {SupportedLiteLLMProvider.Anthropic, SupportedLiteLLMProvider.Bedrock}:+ _apply_claude_sampling_policy(model_name_lower, sanitized_gen_conf, sanitized_kwargs)if provider == SupportedLiteLLMProvider.HunYuan:for key in ("presence_penalty", "frequency_penalty"):diff --git a/test/unit_test/rag/llm/test_chat_model_thinking_policy.py b/test/unit_test/rag/llm/test_chat_model_thinking_policy.pyindex 3931c3a847..afb2f4e22d 100644--- a/test/unit_test/rag/llm/test_chat_model_thinking_policy.py+++ b/test/unit_test/rag/llm/test_chat_model_thinking_policy.py@@ -14,10 +14,12 @@# limitations under the License.#+import logging+import pytestfrom rag.llm import SupportedLiteLLMProvider-from rag.llm.chat_model import _apply_model_family_policies, _move_litellm_provider_body_fields+from rag.llm.chat_model import _apply_claude_sampling_policy, _apply_model_family_policies, _claude_version, _move_litellm_provider_body_fieldspytestmark = pytest.mark.p1@@ -327,3 +329,140 @@ def test_litellm_provider_body_fields_preserve_existing_extra_body():assert completion_args["extra_body"] == {"seed": 1, "enable_thinking": False}assert "enable_thinking" not in completion_args+++CLAUDE_GEN_CONF = {"temperature": 0.8, "top_p": 0.9, "presence_penalty": 0.1, "frequency_penalty": 0.1}+++def _litellm_policies(model_name, provider, gen_conf):+ sanitized, _ = _apply_model_family_policies(model_name, backend="litellm", provider=provider, gen_conf=gen_conf)+ return sanitized+++def test_bedrock_claude_keeps_temperature_and_drops_top_p():+ gen_conf = _litellm_policies("eu.anthropic.claude-sonnet-4-6", SupportedLiteLLMProvider.Bedrock, CLAUDE_GEN_CONF)++ assert gen_conf["temperature"] == 0.8+ assert "top_p" not in gen_conf+ assert gen_conf["presence_penalty"] == 0.1+++def test_bedrock_claude_top_p_alone_is_preserved():+ gen_conf = _litellm_policies("eu.anthropic.claude-sonnet-4-6", SupportedLiteLLMProvider.Bedrock, {"top_p": 0.9})++ assert gen_conf == {"top_p": 0.9}+++ "model_name",+ [+ "eu.anthropic.claude-opus-4-7-v1",+ "eu.anthropic.claude-opus-4-8-v1:0",+ "eu.anthropic.claude-opus-5",+ "eu.anthropic.claude-sonnet-5",+ "anthropic.claude-fable-5-1",+ ],+)+def test_bedrock_claude_without_sampling_support_drops_all_sampling_params(model_name):+ gen_conf = _litellm_policies(model_name, SupportedLiteLLMProvider.Bedrock, {**CLAUDE_GEN_CONF, "top_k": 40})++ assert not {"temperature", "top_p", "top_k"} & set(gen_conf)+ assert gen_conf["presence_penalty"] == 0.1+++def test_claude_sonnet_4_5_is_not_matched_as_sonnet_5():+ gen_conf = _litellm_policies("eu.anthropic.claude-sonnet-4-5", SupportedLiteLLMProvider.Bedrock, CLAUDE_GEN_CONF)++ assert gen_conf["temperature"] == 0.8+ assert "top_p" not in gen_conf+++def test_anthropic_opus_4_8_still_drops_all_sampling_params():+ gen_conf = _litellm_policies("claude-opus-4-8", SupportedLiteLLMProvider.Anthropic, CLAUDE_GEN_CONF)++ assert not {"temperature", "top_p", "top_k"} & set(gen_conf)+++def test_bedrock_non_claude_model_is_untouched():+ gen_conf = _litellm_policies("mistral.mistral-large-2402-v1:0", SupportedLiteLLMProvider.Bedrock, CLAUDE_GEN_CONF)++ assert gen_conf == CLAUDE_GEN_CONF+++def test_claude_sampling_policy_drops_top_p_across_targets_when_temperature_is_anywhere():+ gen_conf, kwargs = {"top_p": 0.9}, {"temperature": 0.2}++ _apply_claude_sampling_policy("eu.anthropic.claude-sonnet-4-6", gen_conf, kwargs)++ assert gen_conf == {}+ assert kwargs == {"temperature": 0.2}+++def test_claude_sampling_policy_logs_applied_policy(caplog):+ with caplog.at_level(logging.DEBUG):+ _apply_claude_sampling_policy("eu.anthropic.claude-sonnet-4-6", {"temperature": 0.8, "top_p": 0.9})+ _apply_claude_sampling_policy("eu.anthropic.claude-opus-4-8-v1:0", {"temperature": 0.8})+ _apply_claude_sampling_policy("eu.anthropic.claude-sonnet-4-6", {"top_p": 0.9})++ records = [record for record in caplog.records if "Claude sampling policy" in record.getMessage()]+ assert [record.levelno for record in records] == [logging.WARNING, logging.WARNING]+ messages = [record.getMessage() for record in records]+ assert "dropped top_p for model eu.anthropic.claude-sonnet-4-6" in messages[0]+ assert "dropped temperature for model eu.anthropic.claude-opus-4-8-v1:0" in messages[1]+++def test_claude_sampling_policy_no_sampling_model_without_params_does_not_log(caplog):+ gen_conf = {"presence_penalty": 0.1}++ with caplog.at_level(logging.DEBUG):+ _apply_claude_sampling_policy("eu.anthropic.claude-opus-4-8-v1:0", gen_conf, {})++ assert gen_conf == {"presence_penalty": 0.1}+ assert not [record for record in caplog.records if "Claude sampling policy" in record.getMessage()]+++ ("model_name", "expected"),+ [+ ("claude-sonnet-4-5", (4, 5)),+ ("claude-sonnet-4-5-20250929", (4, 5)),+ ("claude-opus-4-1-20250805", (4, 1)),+ ("claude-sonnet-4-20250514", (4, 0)),+ ("us.anthropic.claude-sonnet-4-20250514-v1:0", (4, 0)),+ ("eu.anthropic.claude-sonnet-4-6", (4, 6)),+ ("claude-haiku-4-5-20251001", (4, 5)),+ ("claude-fable-5-1", (5, 1)),+ ("claude-3-5-sonnet-20241022", (3, 5)),+ ("anthropic.claude-3-7-sonnet-20250219-v1:0", (3, 7)),+ ("claude-3-haiku-20240307", (3, 0)),+ ("claude-instant-1.2", None),+ ],+)+def test_claude_version_parsing(model_name, expected):+ assert _claude_version(model_name) == expected++[email protected]("provider", [SupportedLiteLLMProvider.Anthropic, SupportedLiteLLMProvider.Bedrock])+ "model_name",+ ["claude-3-7-sonnet-20250219", "claude-3-5-sonnet-20241022", "claude-sonnet-4-20250514", "anthropic.claude-opus-4-20250514-v1:0"],+)+def test_claude_before_4_1_keeps_temperature_and_top_p(model_name, provider):+ gen_conf = _litellm_policies(model_name, provider, CLAUDE_GEN_CONF)++ assert gen_conf == CLAUDE_GEN_CONF++[email protected]("provider", [SupportedLiteLLMProvider.Anthropic, SupportedLiteLLMProvider.Bedrock])[email protected]("model_name", ["claude-opus-4-1-20250805", "claude-sonnet-4-5", "claude-haiku-4-5-20251001", "claude-opus-4-6"])+def test_claude_4_1_and_later_drop_top_p_on_both_providers(model_name, provider):+ gen_conf = _litellm_policies(model_name, provider, CLAUDE_GEN_CONF)++ assert gen_conf["temperature"] == 0.8+ assert "top_p" not in gen_conf+++def test_claude_with_unparsable_version_is_treated_as_recent():+ gen_conf = _litellm_policies("claude-latest", SupportedLiteLLMProvider.Anthropic, CLAUDE_GEN_CONF)++ assert "top_p" not in gen_conf