diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs index d43060cf0..1a0627147 100644 --- a/crates/switchyard-server/src/config.rs +++ b/crates/switchyard-server/src/config.rs @@ -21,7 +21,7 @@ use switchyard_llm_client::{ }; use switchyard_protocol::RoutedLlmClient; -use crate::{ServerError, ServerResult, ServerState}; +use crate::{ModelCapabilities, ServerError, ServerResult, ServerState}; const SUPPORTED_SCHEMA_VERSION: u32 = 1; const MAX_CONFIGURED_RETRIES: u32 = 10; @@ -91,12 +91,16 @@ impl ServerConfig { for (route_name, config) in &self.routes { validate_value("route name", route_name)?; validate_value(&format!("route {route_name} id"), config.id())?; - routes.push(( - config.id().to_string(), - build_algorithm(route_name, config, &targets)?, - )); + let capabilities = config.capabilities(); + if capabilities.context_window == Some(0) { + return Err(ServerError::new(format!( + "route {route_name} context_window must be greater than zero" + ))); + } + let algorithm = build_algorithm(route_name, config, &targets)?; + routes.push((config.id().to_string(), algorithm, capabilities)); } - ServerState::new(routes) + ServerState::new_with_capabilities(routes) } fn build_clients(&self) -> ServerResult>> { @@ -260,19 +264,35 @@ struct CustomClassifierRouteConfig { enum RouteConfig { Noop { id: String, + #[serde(default)] + context_window: Option, + #[serde(default)] + tool_calling: Option, }, Random { id: String, + #[serde(default)] + context_window: Option, + #[serde(default)] + tool_calling: Option, targets: Vec, weights: Option>, seed: Option, }, Passthrough { id: String, + #[serde(default)] + context_window: Option, + #[serde(default)] + tool_calling: Option, target: String, }, LlmClassifier { id: String, + #[serde(default)] + context_window: Option, + #[serde(default)] + tool_calling: Option, classifier_target: String, #[serde(default)] mode: Option, @@ -309,6 +329,10 @@ enum RouteConfig { }, StageRouter { id: String, + #[serde(default)] + context_window: Option, + #[serde(default)] + tool_calling: Option, capable_target: String, efficient_target: String, /// Tier a turn falls back to when the signals are not confident. @@ -373,7 +397,7 @@ impl RouteConfig { fn id(&self) -> &str { use RouteConfig::*; match self { - Noop { id } + Noop { id, .. } | Random { id, .. } | LlmClassifier { id, .. } | Passthrough { id, .. } @@ -381,6 +405,40 @@ impl RouteConfig { } } + fn capabilities(&self) -> ModelCapabilities { + use RouteConfig::*; + match self { + Noop { + context_window, + tool_calling, + .. + } + | Random { + context_window, + tool_calling, + .. + } + | Passthrough { + context_window, + tool_calling, + .. + } + | LlmClassifier { + context_window, + tool_calling, + .. + } + | StageRouter { + context_window, + tool_calling, + .. + } => ModelCapabilities { + context_window: *context_window, + tool_calling: *tool_calling, + }, + } + } + fn classifier_mode(&self, route_name: &str) -> ServerResult { let Self::LlmClassifier { mode, @@ -997,6 +1055,12 @@ target = "weak" ); assert!(error_message(&unknown_classifier_field).contains("unknown field")); + let target_capability = VALID_CONFIG.replace( + "llm_client = \"responses\"", + "llm_client = \"responses\"\ncontext_window = 1000000", + ); + assert!(error_message(&target_capability).contains("unknown field `context_window`")); + let unknown_algorithm = VALID_CONFIG.replace("type = \"noop\"", "type = \"imaginary\""); assert!(error_message(&unknown_algorithm).contains("unknown variant")); } @@ -1090,6 +1154,13 @@ classifier_magic = true VALID_CONFIG.replace("[targets.strong]", "[targets.\" strong \"]"), "target name must be non-empty and have no surrounding whitespace", ), + ( + VALID_CONFIG.replace( + "targets = [\"strong\", \"weak\"]", + "targets = [\"strong\", \"weak\"]\ncontext_window = 0", + ), + "route random context_window must be greater than zero", + ), ]; for (toml, expected) in cases { diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index 97c253b4d..9196f0844 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -84,10 +84,27 @@ impl Error for ServerError {} /// Result returned by server setup and lifecycle operations. pub type ServerResult = std::result::Result; +/// Capabilities that one route advertises on `GET /v1/models`. +/// +/// An unset capability is undeclared and serializes as `null`. +#[derive(Clone, Copy, Default)] +struct ModelCapabilities { + context_window: Option, + tool_calling: Option, +} + +/// A registered route: the libsy algorithm that serves it and the capabilities +/// advertised for it on `GET /v1/models`. One entry owns both so the routing +/// runtime and the model listing can never drift apart. +struct RouteEntry { + algorithm: Arc, + capabilities: ModelCapabilities, +} + /// Shared server state used by all endpoint handlers. #[derive(Clone)] pub struct ServerState { - routes: Arc>>, + routes: Arc>, metrics: prometheus::Registry, stats: StatsAccumulator, routing_log: Option, @@ -132,14 +149,28 @@ impl ServerState { /// Creates server state from route model IDs and their libsy algorithms. pub fn new( routes: impl IntoIterator)>, + ) -> ServerResult { + Self::new_with_capabilities( + routes + .into_iter() + .map(|(model, algorithm)| (model, algorithm, ModelCapabilities::default())), + ) + } + + fn new_with_capabilities( + routes: impl IntoIterator, ModelCapabilities)>, ) -> ServerResult { let mut entries = BTreeMap::new(); - for (model, algorithm) in routes { + for (model, algorithm, capabilities) in routes { let model = model.trim(); if model.is_empty() { return Err(ServerError::new("route model must not be empty")); } - if entries.insert(model.to_string(), algorithm).is_some() { + let entry = RouteEntry { + algorithm, + capabilities, + }; + if entries.insert(model.to_string(), entry).is_some() { return Err(ServerError::new(format!("duplicate route model {model}"))); } } @@ -168,7 +199,9 @@ impl ServerState { } fn algorithm_for_model(&self, model: &str) -> Option> { - self.routes.get(model).map(Arc::clone) + self.routes + .get(model) + .map(|entry| Arc::clone(&entry.algorithm)) } } @@ -827,7 +860,12 @@ fn error_response( } async fn models(State(state): State) -> Json { - Json(model_list_payload(state.models())) + Json(model_list_payload( + state + .routes + .iter() + .map(|(model, entry)| (model.as_str(), entry.capabilities)), + )) } async fn get_stats(State(state): State) -> Json { @@ -907,13 +945,16 @@ async fn not_found() -> Response { ) } -fn model_list_payload<'a>(models: impl IntoIterator) -> Value { - let model_ids = models.into_iter().map(str::to_string).collect::>(); - let first_id = model_ids.first().cloned(); - let last_id = model_ids.last().cloned(); +fn model_list_payload<'a>( + entries: impl IntoIterator, +) -> Value { + let entries = entries.into_iter().collect::>(); + let model_ids = entries.iter().map(|(model, _)| *model).collect::>(); + let first_id = model_ids.first().copied(); + let last_id = model_ids.last().copied(); json!({ "object": "list", - "data": model_ids.iter().map(|model| model_entry_json(model)).collect::>(), + "data": entries.iter().map(|(model, caps)| model_entry_json(model, *caps)).collect::>(), "first_id": first_id, "last_id": last_id, "has_more": false, @@ -922,7 +963,7 @@ fn model_list_payload<'a>(models: impl IntoIterator) -> Value { }) } -fn model_entry_json(model: &str) -> Value { +fn model_entry_json(model: &str, capabilities: ModelCapabilities) -> Value { json!({ "id": model, "object": "model", @@ -932,8 +973,8 @@ fn model_entry_json(model: &str) -> Value { "display_name": model, "capabilities": { "streaming": true, - "tool_calling": null, - "context_window": null, + "tool_calling": capabilities.tool_calling, + "context_window": capabilities.context_window, "supported_inbound_formats": [ "openai-chat-completions", "openai-responses", diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 406110db9..3860cdc85 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -1101,6 +1101,58 @@ async fn routes_dispatch_and_discovery_endpoints_are_stable() -> TestResult { Ok(()) } +#[tokio::test] +async fn models_endpoint_reports_declared_route_capabilities_and_null_when_undeclared() -> TestResult +{ + const CONFIG: &str = r#" +schema_version = 1 + +[llm_clients.primary] +format = "openai_chat" +base_url = "https://example.test/v1" + +[targets.shared] +id = "nvidia/deepseek-ai/deepseek-v4-pro" +llm_client = "primary" + +[routes.declared] +id = "declared" +type = "passthrough" +target = "shared" +context_window = 1000000 +tool_calling = true + +[routes.restricted] +id = "restricted" +type = "passthrough" +target = "shared" +context_window = 262000 +tool_calling = false + +[routes.undeclared] +id = "undeclared" +type = "passthrough" +target = "shared" +"#; + let app = build_switchyard_router(load_test_config(CONFIG)?); + let models = send(&app, "GET", "/v1/models", None).await?; + assert_eq!(models.status, StatusCode::OK); + let body = models.json()?; + let data = body["data"].as_array().cloned().unwrap_or_default(); + let capabilities = data + .iter() + .filter_map(|entry| entry["id"].as_str().map(|id| (id, &entry["capabilities"]))) + .collect::>(); + + assert_eq!(capabilities["declared"]["context_window"], json!(1_000_000)); + assert_eq!(capabilities["declared"]["tool_calling"], json!(true)); + assert_eq!(capabilities["restricted"]["context_window"], json!(262_000)); + assert_eq!(capabilities["restricted"]["tool_calling"], json!(false)); + assert_eq!(capabilities["undeclared"]["context_window"], json!(null)); + assert_eq!(capabilities["undeclared"]["tool_calling"], json!(null)); + Ok(()) +} + #[tokio::test] async fn all_inbound_formats_run_libsy_and_return_the_caller_format() -> TestResult { let (upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?; diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 75e4eac9d..45092727e 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -56,7 +56,14 @@ must exist and be non-empty when the server loads. ## `[routes.]` -Every route takes `id` and `type`, plus the keys for that type. +Every route takes the common keys below, plus the keys for its type. + +| Key | Required | Default | Meaning | +|---|:---:|---|---| +| `id` | Yes | — | Public model ID that callers send in requests. | +| `type` | Yes | — | Routing algorithm for this route. | +| `context_window` | No | unset | Positive token count advertised for this route by `GET /v1/models`. Unset values appear as `null`. This does not enforce a request limit. | +| `tool_calling` | No | unset | Whether `GET /v1/models` advertises tool-calling support for this route. Unset values appear as `null`. | ### `noop`