From 15e8ec1a4c06d8afdb3d80ca093d6896207aaf40 Mon Sep 17 00:00:00 2001 From: Hiroshi Morishige Date: Thu, 23 Jul 2026 19:05:30 +0900 Subject: [PATCH] fix(profiles): gate DeepSeek thinking-off default on reasoning-hint compatibility apply_deepseek_overrides() unconditionally injects the vLLM-only chat_template_kwargs.enable_thinking=false hint for any target whose model id contains "deepseek-v4". OpenAI-compatible providers with strict request validation reject the unknown field with HTTP 400 ("Extra inputs are not permitted"), which breaks every routed call for deepseek-v4 tiers served outside the NVIDIA benchmark gateway (observed on Fireworks AI serverless). Gate the body default on model_accepts_reasoning_hint(), the same model-id compatibility check the LLM classifier already applies before sending the hint on its own calls. The provider-neutral X-Inference-Priority header default is unchanged, and explicitly provided extra_body still wins as before. Signed-off-by: Hiroshi Morishige --- switchyard/lib/profiles/tier_target_builders.py | 13 +++++++++++-- tests/test_deterministic_routing_profile.py | 15 +++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/switchyard/lib/profiles/tier_target_builders.py b/switchyard/lib/profiles/tier_target_builders.py index 4b87c56ac..3ad5768c1 100644 --- a/switchyard/lib/profiles/tier_target_builders.py +++ b/switchyard/lib/profiles/tier_target_builders.py @@ -20,6 +20,7 @@ build_native_backend, resolve_llm_target, ) +from switchyard.lib.processors.reasoning_hint import model_accepts_reasoning_hint from switchyard.lib.profiles.deterministic_routing_config import ( DEFAULT_DETERMINISTIC_TIER_TIMEOUT_S, ) @@ -27,10 +28,18 @@ def apply_deepseek_overrides(target: LlmTarget) -> LlmTarget: - """Apply benchmark-specific DeepSeek extras without clobbering callers.""" + """Apply benchmark-specific DeepSeek extras without clobbering callers. + + The thinking-off default is a vLLM-side hint (``chat_template_kwargs``); + serving stacks on the :func:`model_accepts_reasoning_hint` deny list + reject the field outright (HTTP 400 ``Extra inputs are not permitted``), + so the default is gated on the same model-id check the LLM classifier + already uses for its own calls. The batch-priority header default is + provider-neutral and stays unconditional. + """ default_body = ( {"chat_template_kwargs": {"enable_thinking": False}} - if "deepseek-v4" in target.model + if "deepseek-v4" in target.model and model_accepts_reasoning_hint(target.model) else None ) default_headers = ( diff --git a/tests/test_deterministic_routing_profile.py b/tests/test_deterministic_routing_profile.py index 331b3bb21..c3cbd55e7 100644 --- a/tests/test_deterministic_routing_profile.py +++ b/tests/test_deterministic_routing_profile.py @@ -391,6 +391,21 @@ def test_caller_supplied_empty_body_wins(self) -> None: out = apply_deepseek_overrides(target) assert out.extra_body == {} + def test_hint_incompatible_model_id_skips_thinking_off(self) -> None: + """Deny-listed serving stacks reject the vLLM hint with HTTP 400, + so the thinking-off default must not be injected for their ids. + The provider-neutral batch-priority header still applies.""" + target = LlmTarget( + id="weak", + model="bedrock/deepseek-ai/deepseek-v4-flash", + format=BackendFormat.OPENAI, + api_key="k", + base_url="https://e/v1", + ) + out = apply_deepseek_overrides(target) + assert not out.extra_body + assert out.extra_headers == {"X-Inference-Priority": "batch"} + class TestTierTimeoutDefaults: """Deterministic tiers get a bounded timeout unless callers set one."""