Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(llm): drop Claude sampling params rejected by Bedrock and Anthropic (#19295)

VincentLambert2b8861ceb899
diff --git a/rag/llm/chat_model.py b/rag/llm/chat_model.py
index 5b5870c6ce..d9ae2716bb 100644
--- a/rag/llm/chat_model.py
+++ b/rag/llm/chat_model.py
@@ -112,6 +112,68 @@ LITELLM_ALLOWED_GEN_CONF_KEYS = ALLOWED_GEN_CONF_KEYS | frozenset(
}
)
+# Claude models that reject every sampling parameter (temperature / top_p / top_k -> HTTP 400).
+_CLAUDE_NO_SAMPLING_MARKERS = (
+ "claude-opus-4-7",
+ "claude-opus-4-8",
+ "claude-opus-5",
+ "claude-sonnet-5",
+ "claude-fable",
+ "claude-mythos",
+)
+
+# First Claude generation that rejects ``temperature`` and ``top_p`` being set together, on the
+# Anthropic API itself as well as through Bedrock (Anthropic API release notes, 2025-08-05:
+# "Opus 4.1 does not allow both temperature and top_p parameters to be specified"; Haiku 4.5
+# migration guide: "Use only temperature OR top_p, not both. Setting both returns a 400 error").
+# Claude 3.x and Claude 4.0 (Opus 4 / Sonnet 4) still accept the pair.
+_CLAUDE_TEMPERATURE_XOR_TOP_P_SINCE = (4, 1)
+
+# ``claude-sonnet-4-5[-20250929]``, ``eu.anthropic.claude-opus-4-1-20250805-v1:0``, ``claude-fable-5-1``.
+# The minor version is at most two digits so a date suffix (``claude-sonnet-4-20250514``) is not read as one.
+_CLAUDE_VERSION_RE = re.compile(r"claude-(?:opus|sonnet|haiku|fable|mythos)-(\d+)(?:-(\d{1,2}))?(?!\d)")
+# Legacy naming: ``claude-3-5-sonnet-20241022``, ``anthropic.claude-3-7-sonnet-20250219-v1:0``, ``claude-3-haiku``.
+_CLAUDE_LEGACY_VERSION_RE = re.compile(r"claude-(\d)(?:-(\d))?-(?:opus|sonnet|haiku)")
+
+
+def _claude_version(model_name_lower: str) -> tuple[int, int] | None:
+ """Return the ``(major, minor)`` Claude generation parsed from a model name, or ``None`` when unknown."""
+ match = _CLAUDE_VERSION_RE.search(model_name_lower) or _CLAUDE_LEGACY_VERSION_RE.search(model_name_lower)
+ if not match:
+ return None
+ return int(match.group(1)), int(match.group(2) or 0)
+
+
+def _apply_claude_sampling_policy(model_name_lower: str, *targets: dict) -> None:
+ """Drop, in place on every ``targets`` dict, the sampling parameters a Claude model would reject.
+
+ The rules are properties of the model generation, not of the gateway, so they apply to Claude
+ served by Anthropic directly *and* through Bedrock (``eu.anthropic.claude-sonnet-4-6``):
+
+ * Opus 4.7+, Sonnet 5, Fable/Mythos: no sampling parameter is accepted at all.
+ * Claude 4.1 and later: ``temperature`` and ``top_p`` cannot both be specified (HTTP 400,
+ "temperature and top_p cannot both be specified for this model"). ``temperature`` is the
+ primary UI knob, so ``top_p`` is the one dropped.
+ * Claude 3.x and 4.0 accept the pair and are left untouched. A Claude name whose generation
+ cannot be parsed is treated as recent, since a dropped ``top_p`` beats a failing chat.
+
+ Every drop is logged at WARNING level so the user can find out why a setting is not honoured.
+ """
+ if "claude" not in model_name_lower:
+ return
+ if any(marker in model_name_lower for marker in _CLAUDE_NO_SAMPLING_MARKERS):
+ removed = [key for target in targets for key in ("temperature", "top_p", "top_k") if target.pop(key, None) is not None]
+ if removed:
+ logging.warning("Claude sampling policy: dropped %s for model %s (no sampling parameter accepted)", "/".join(sorted(set(removed))), model_name_lower)
+ return
+ version = _claude_version(model_name_lower)
+ if version is not None and version < _CLAUDE_TEMPERATURE_XOR_TOP_P_SINCE:
+ return
+ if any("temperature" in target for target in targets) and any("top_p" in target for target in targets):
+ for target in targets:
+ target.pop("top_p", None)
+ logging.warning("Claude sampling policy: dropped top_p for model %s (temperature and top_p cannot both be specified)", model_name_lower)
+
def _apply_model_family_policies(
model_name: str,
@@ -204,10 +266,8 @@ def _apply_model_family_policies(
for key in ("temperature", "top_p", "logprobs", "top_logprobs"):
sanitized_gen_conf.pop(key, None)
sanitized_kwargs.pop(key, None)
- elif provider == SupportedLiteLLMProvider.Anthropic and model_name_lower in {"claude-opus-4-7", "claude-opus-4-8"}:
- for key in ("temperature", "top_p", "top_k"):
- sanitized_gen_conf.pop(key, None)
- sanitized_kwargs.pop(key, None)
+ elif provider in {SupportedLiteLLMProvider.Anthropic, SupportedLiteLLMProvider.Bedrock}:
+ _apply_claude_sampling_policy(model_name_lower, sanitized_gen_conf, sanitized_kwargs)
if provider == SupportedLiteLLMProvider.HunYuan:
for key in ("presence_penalty", "frequency_penalty"):
diff --git a/test/unit_test/rag/llm/test_chat_model_thinking_policy.py b/test/unit_test/rag/llm/test_chat_model_thinking_policy.py
index 3931c3a847..afb2f4e22d 100644
--- a/test/unit_test/rag/llm/test_chat_model_thinking_policy.py
+++ b/test/unit_test/rag/llm/test_chat_model_thinking_policy.py
@@ -14,10 +14,12 @@
# limitations under the License.
#
+import logging
+
import pytest
from rag.llm import SupportedLiteLLMProvider
-from rag.llm.chat_model import _apply_model_family_policies, _move_litellm_provider_body_fields
+from rag.llm.chat_model import _apply_claude_sampling_policy, _apply_model_family_policies, _claude_version, _move_litellm_provider_body_fields
pytestmark = pytest.mark.p1
@@ -327,3 +329,140 @@ def test_litellm_provider_body_fields_preserve_existing_extra_body():
assert completion_args["extra_body"] == {"seed": 1, "enable_thinking": False}
assert "enable_thinking" not in completion_args
+
+
+CLAUDE_GEN_CONF = {"temperature": 0.8, "top_p": 0.9, "presence_penalty": 0.1, "frequency_penalty": 0.1}
+
+
+def _litellm_policies(model_name, provider, gen_conf):
+ sanitized, _ = _apply_model_family_policies(model_name, backend="litellm", provider=provider, gen_conf=gen_conf)
+ return sanitized
+
+
+def test_bedrock_claude_keeps_temperature_and_drops_top_p():
+ gen_conf = _litellm_policies("eu.anthropic.claude-sonnet-4-6", SupportedLiteLLMProvider.Bedrock, CLAUDE_GEN_CONF)
+
+ assert gen_conf["temperature"] == 0.8
+ assert "top_p" not in gen_conf
+ assert gen_conf["presence_penalty"] == 0.1
+
+
+def test_bedrock_claude_top_p_alone_is_preserved():
+ gen_conf = _litellm_policies("eu.anthropic.claude-sonnet-4-6", SupportedLiteLLMProvider.Bedrock, {"top_p": 0.9})
+
+ assert gen_conf == {"top_p": 0.9}
+
+
+ "model_name",
+ [
+ "eu.anthropic.claude-opus-4-7-v1",
+ "eu.anthropic.claude-opus-4-8-v1:0",
+ "eu.anthropic.claude-opus-5",
+ "eu.anthropic.claude-sonnet-5",
+ "anthropic.claude-fable-5-1",
+ ],
+)
+def test_bedrock_claude_without_sampling_support_drops_all_sampling_params(model_name):
+ gen_conf = _litellm_policies(model_name, SupportedLiteLLMProvider.Bedrock, {**CLAUDE_GEN_CONF, "top_k": 40})
+
+ assert not {"temperature", "top_p", "top_k"} & set(gen_conf)
+ assert gen_conf["presence_penalty"] == 0.1
+
+
+def test_claude_sonnet_4_5_is_not_matched_as_sonnet_5():
+ gen_conf = _litellm_policies("eu.anthropic.claude-sonnet-4-5", SupportedLiteLLMProvider.Bedrock, CLAUDE_GEN_CONF)
+
+ assert gen_conf["temperature"] == 0.8
+ assert "top_p" not in gen_conf
+
+
+def test_anthropic_opus_4_8_still_drops_all_sampling_params():
+ gen_conf = _litellm_policies("claude-opus-4-8", SupportedLiteLLMProvider.Anthropic, CLAUDE_GEN_CONF)
+
+ assert not {"temperature", "top_p", "top_k"} & set(gen_conf)
+
+
+def test_bedrock_non_claude_model_is_untouched():
+ gen_conf = _litellm_policies("mistral.mistral-large-2402-v1:0", SupportedLiteLLMProvider.Bedrock, CLAUDE_GEN_CONF)
+
+ assert gen_conf == CLAUDE_GEN_CONF
+
+
+def test_claude_sampling_policy_drops_top_p_across_targets_when_temperature_is_anywhere():
+ gen_conf, kwargs = {"top_p": 0.9}, {"temperature": 0.2}
+
+ _apply_claude_sampling_policy("eu.anthropic.claude-sonnet-4-6", gen_conf, kwargs)
+
+ assert gen_conf == {}
+ assert kwargs == {"temperature": 0.2}
+
+
+def test_claude_sampling_policy_logs_applied_policy(caplog):
+ with caplog.at_level(logging.DEBUG):
+ _apply_claude_sampling_policy("eu.anthropic.claude-sonnet-4-6", {"temperature": 0.8, "top_p": 0.9})
+ _apply_claude_sampling_policy("eu.anthropic.claude-opus-4-8-v1:0", {"temperature": 0.8})
+ _apply_claude_sampling_policy("eu.anthropic.claude-sonnet-4-6", {"top_p": 0.9})
+
+ records = [record for record in caplog.records if "Claude sampling policy" in record.getMessage()]
+ assert [record.levelno for record in records] == [logging.WARNING, logging.WARNING]
+ messages = [record.getMessage() for record in records]
+ assert "dropped top_p for model eu.anthropic.claude-sonnet-4-6" in messages[0]
+ assert "dropped temperature for model eu.anthropic.claude-opus-4-8-v1:0" in messages[1]
+
+
+def test_claude_sampling_policy_no_sampling_model_without_params_does_not_log(caplog):
+ gen_conf = {"presence_penalty": 0.1}
+
+ with caplog.at_level(logging.DEBUG):
+ _apply_claude_sampling_policy("eu.anthropic.claude-opus-4-8-v1:0", gen_conf, {})
+
+ assert gen_conf == {"presence_penalty": 0.1}
+ assert not [record for record in caplog.records if "Claude sampling policy" in record.getMessage()]
+
+
+ ("model_name", "expected"),
+ [
+ ("claude-sonnet-4-5", (4, 5)),
+ ("claude-sonnet-4-5-20250929", (4, 5)),
+ ("claude-opus-4-1-20250805", (4, 1)),
+ ("claude-sonnet-4-20250514", (4, 0)),
+ ("us.anthropic.claude-sonnet-4-20250514-v1:0", (4, 0)),
+ ("eu.anthropic.claude-sonnet-4-6", (4, 6)),
+ ("claude-haiku-4-5-20251001", (4, 5)),
+ ("claude-fable-5-1", (5, 1)),
+ ("claude-3-5-sonnet-20241022", (3, 5)),
+ ("anthropic.claude-3-7-sonnet-20250219-v1:0", (3, 7)),
+ ("claude-3-haiku-20240307", (3, 0)),
+ ("claude-instant-1.2", None),
+ ],
+)
+def test_claude_version_parsing(model_name, expected):
+ assert _claude_version(model_name) == expected
+
+
[email protected]("provider", [SupportedLiteLLMProvider.Anthropic, SupportedLiteLLMProvider.Bedrock])
+ "model_name",
+ ["claude-3-7-sonnet-20250219", "claude-3-5-sonnet-20241022", "claude-sonnet-4-20250514", "anthropic.claude-opus-4-20250514-v1:0"],
+)
+def test_claude_before_4_1_keeps_temperature_and_top_p(model_name, provider):
+ gen_conf = _litellm_policies(model_name, provider, CLAUDE_GEN_CONF)
+
+ assert gen_conf == CLAUDE_GEN_CONF
+
+
[email protected]("provider", [SupportedLiteLLMProvider.Anthropic, SupportedLiteLLMProvider.Bedrock])
[email protected]("model_name", ["claude-opus-4-1-20250805", "claude-sonnet-4-5", "claude-haiku-4-5-20251001", "claude-opus-4-6"])
+def test_claude_4_1_and_later_drop_top_p_on_both_providers(model_name, provider):
+ gen_conf = _litellm_policies(model_name, provider, CLAUDE_GEN_CONF)
+
+ assert gen_conf["temperature"] == 0.8
+ assert "top_p" not in gen_conf
+
+
+def test_claude_with_unparsable_version_is_treated_as_recent():
+ gen_conf = _litellm_policies("claude-latest", SupportedLiteLLMProvider.Anthropic, CLAUDE_GEN_CONF)
+
+ assert "top_p" not in gen_conf