diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs index 6ba089d9c..2a6948605 100644 --- a/crates/libsy-llm-client/src/client.rs +++ b/crates/libsy-llm-client/src/client.rs @@ -501,6 +501,7 @@ impl TranslatingLlmClient { } let retry_after = retry_after_delay(response.headers()); + let headers = Box::new(safe_upstream_error_headers(response.headers())); let body = match response.text().await { Ok(body) => body, Err(error) => { @@ -521,7 +522,11 @@ impl TranslatingLlmClient { message: body, } } else { - LlmClientError::UpstreamHttp { status, body } + LlmClientError::UpstreamHttp { + status, + body, + headers, + } }; Err(AttemptFailure { error, @@ -610,6 +615,7 @@ impl TranslatingLlmClient { metadata.as_ref(), backend.is_forwarding_auth(), ), + headers: Box::default(), } } error => LlmClientError::ResponseTranslation(error.to_string()), @@ -852,6 +858,21 @@ fn first_event_overflow( }) } +/// Retain end-to-end retry and correlation headers from a failed upstream response. +fn safe_upstream_error_headers(headers: &HeaderMap) -> HeaderMap { + let mut safe = HeaderMap::new(); + for (name, value) in headers { + let name_str = name.as_str(); + let allowed = matches!(name_str, "retry-after" | "request-id" | "x-request-id") + || name_str.starts_with("x-ratelimit-") + || name_str.starts_with("anthropic-ratelimit-"); + if allowed && !is_non_forwardable_header(name_str, headers) { + safe.append(name.clone(), value.clone()); + } + } + safe +} + // Uses Retry-After when supplied, capped so an upstream cannot stall a request indefinitely. fn retry_after_delay(headers: &HeaderMap) -> Option { let value = headers.get(RETRY_AFTER)?.to_str().ok()?; @@ -2433,7 +2454,8 @@ mod tests { error, LlmClientError::UpstreamHttp { status: StatusCode::UNAUTHORIZED, - body + body, + .. } if body == "invalid key" )); assert_eq!(calls.load(Ordering::SeqCst), 1); @@ -2496,6 +2518,8 @@ mod tests { let attempt = observed_calls.fetch_add(1, Ordering::SeqCst) + 1; ResponseTemplate::new(500) .insert_header("retry-after", "0") + .insert_header("x-request-id", format!("attempt-{attempt}").as_str()) + .insert_header("set-cookie", "session=secret") .set_body_string(format!("attempt {attempt}")) }) .mount(&server) @@ -2510,17 +2534,51 @@ mod tests { panic!("expected retry exhaustion"); }; - assert!(matches!( - error, - LlmClientError::UpstreamHttp { - status: StatusCode::INTERNAL_SERVER_ERROR, - body - } if body == "attempt 3" - )); + let LlmClientError::UpstreamHttp { + status, + body, + headers, + } = error + else { + panic!("expected final upstream error"); + }; + assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR); + assert_eq!(body, "attempt 3"); + assert_eq!(headers.get("x-request-id"), Some(&"attempt-3".parse()?)); + assert!(!headers.contains_key("set-cookie")); assert_eq!(calls.load(Ordering::SeqCst), 3); Ok(()) } + /// Connection-nominated headers must not survive the upstream error allowlist. + #[test] + fn upstream_error_headers_exclude_connection_nominations() { + let mut headers = HeaderMap::new(); + headers.append( + "connection", + "x-request-id, x-ratelimit-reset-requests" + .parse() + .expect("header value"), + ); + headers.append("connection", "retry-after".parse().expect("header value")); + headers.insert("x-request-id", "hop-id".parse().expect("header value")); + headers.insert( + "x-ratelimit-reset-requests", + "15s".parse().expect("header value"), + ); + headers.insert("retry-after", "15".parse().expect("header value")); + headers.insert("request-id", "end-to-end-id".parse().expect("header value")); + + let safe = safe_upstream_error_headers(&headers); + assert!(!safe.contains_key("x-request-id")); + assert!(!safe.contains_key("x-ratelimit-reset-requests")); + assert!(!safe.contains_key("retry-after")); + assert_eq!( + safe.get("request-id"), + Some(&"end-to-end-id".parse().expect("header value")) + ); + } + #[tokio::test] async fn deadline_expires_before_send_or_stream_poll() -> std::result::Result<(), Box> { @@ -2624,6 +2682,7 @@ mod tests { error: LlmClientError::UpstreamHttp { status, body: String::new(), + headers: Box::default(), }, status: Some(status), retry_after: None, @@ -2641,6 +2700,7 @@ mod tests { error: LlmClientError::UpstreamHttp { status, body: String::new(), + headers: Box::default(), }, status: Some(status), retry_after: None, @@ -2768,6 +2828,7 @@ mod tests { let LlmClientError::UpstreamHttp { status, body: actual, + .. } = error else { panic!("expected the upstream HTTP error, got {error:?}"); diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs index a3dfc339a..5eadb349d 100644 --- a/crates/libsy-llm-client/src/run.rs +++ b/crates/libsy-llm-client/src/run.rs @@ -355,7 +355,7 @@ async fn call_one( result } -// Preserve signed provider events while checking the complete routing response. +/// Preserve signed provider events while checking the complete routing response. async fn buffer_routing_stream( mut chunks: LlmResponseStream, ) -> std::result::Result { @@ -370,6 +370,7 @@ async fn buffer_routing_stream( return Err(LlmClientError::UpstreamHttp { status: StatusCode::BAD_GATEWAY, body: message.clone(), + headers: Box::default(), }); } _ => {} @@ -392,7 +393,7 @@ fn fallback_reason(error: &LibsyError) -> Option { } // A policy denial can be specific to one provider. Preserve its HTTP // error, but allow another candidate to serve the request. - LlmClientError::UpstreamHttp { status, body } + LlmClientError::UpstreamHttp { status, body, .. } if *status == StatusCode::BAD_REQUEST && serde_json::from_str::(body).is_ok_and(|value| { value["error"]["code"].as_str() == Some("content_policy_violation") @@ -1016,10 +1017,12 @@ mod tests { FirstOutcome::ContentPolicy => Err(LlmClientError::UpstreamHttp { status: StatusCode::BAD_REQUEST, body: r#"{"error":{"code":"content_policy_violation","message":"request blocked by content policy","type":"invalid_request_error"},"metadata":{"documentation_section":"context window"}}"#.to_string(), + headers: Box::default(), }), FirstOutcome::Unauthorized => Err(LlmClientError::UpstreamHttp { status: StatusCode::UNAUTHORIZED, body: "unauthorized".to_string(), + headers: Box::default(), }), FirstOutcome::StreamSuccess => Ok(stream_response(vec![ LlmResponseChunk::TextDelta { @@ -1924,6 +1927,7 @@ mod tests { fallback_reason(&error(LlmClientError::UpstreamHttp { status, body: "failed".to_string(), + headers: Box::default(), })), Some(RoutingFallbackReason::Unavailable) ); @@ -1944,6 +1948,7 @@ mod tests { fallback_reason(&error(LlmClientError::UpstreamHttp { status: StatusCode::BAD_REQUEST, body: body.to_string(), + headers: Box::default(), })), expected, "{body}" @@ -1961,6 +1966,7 @@ mod tests { fallback_reason(&error(LlmClientError::UpstreamHttp { status, body: "failed".to_string(), + headers: Box::default(), })), None ); diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rs index 8f5071ccd..2eda791d7 100644 --- a/crates/libsy-llm-client/tests/observability.rs +++ b/crates/libsy-llm-client/tests/observability.rs @@ -449,6 +449,7 @@ impl RoutedLlmClient for JudgeClient { JudgeOutcome::CallFailure => Err(LlmClientError::UpstreamHttp { status: http::StatusCode::INTERNAL_SERVER_ERROR, body: format!(r#"{{"error":{{"message":"server error: {LEAKED_CONTENT}"}}}}"#), + headers: Box::default(), }), JudgeOutcome::Reply(text) => Ok(Response { llm_response: LlmResponse::Agg(text_response(None, *text)), @@ -1575,7 +1576,7 @@ async fn upstream_body_is_redacted_from_the_client_call_span() -> switchyard_lib assert!(matches!( result, Err(LibsyError::ClientCall { - source: LlmClientError::UpstreamHttp { status, body }, + source: LlmClientError::UpstreamHttp { status, body, .. }, .. }) if status == http::StatusCode::INTERNAL_SERVER_ERROR && body.contains(LEAKED_CONTENT) )); diff --git a/crates/libsy/src/algorithms/advisor_gate/tests.rs b/crates/libsy/src/algorithms/advisor_gate/tests.rs index 8bf901595..70a8c7020 100644 --- a/crates/libsy/src/algorithms/advisor_gate/tests.rs +++ b/crates/libsy/src/algorithms/advisor_gate/tests.rs @@ -718,6 +718,7 @@ async fn fail_open_logs_redact_the_upstream_error_body() { Err(LlmClientError::UpstreamHttp { status: http::StatusCode::INTERNAL_SERVER_ERROR, body: format!("{MARKER}: validation failed"), + headers: Box::default(), }) } else { Ok(reply("done")) diff --git a/crates/libsy/src/algorithms/util/buffered_response.rs b/crates/libsy/src/algorithms/util/buffered_response.rs index 706a117d6..897374f79 100644 --- a/crates/libsy/src/algorithms/util/buffered_response.rs +++ b/crates/libsy/src/algorithms/util/buffered_response.rs @@ -70,6 +70,7 @@ pub(crate) async fn buffer_response( Some(LlmClientError::UpstreamHttp { status: http::StatusCode::BAD_GATEWAY, body: message.clone(), + headers: Box::default(), }) } chunk => { diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rs index 9da6589c4..87c19e0ea 100644 --- a/crates/libsy/src/algorithms/util/llm_judge.rs +++ b/crates/libsy/src/algorithms/util/llm_judge.rs @@ -684,6 +684,7 @@ mod tests { LlmClientError::UpstreamHttp { status: StatusCode::INTERNAL_SERVER_ERROR, body: "server error".to_string(), + headers: Box::default(), }, "upstream_5xx", ), @@ -691,6 +692,7 @@ mod tests { LlmClientError::UpstreamHttp { status: StatusCode::FOUND, body: "redirect".to_string(), + headers: Box::default(), }, "upstream_non_5xx", ), diff --git a/crates/libsy/src/algorithms/util/robustness.rs b/crates/libsy/src/algorithms/util/robustness.rs index ed51518a2..67b4f60ec 100644 --- a/crates/libsy/src/algorithms/util/robustness.rs +++ b/crates/libsy/src/algorithms/util/robustness.rs @@ -90,6 +90,7 @@ mod tests { source: LlmClientError::UpstreamHttp { status: http::StatusCode::BAD_REQUEST, body: format!(r#"{{"error":{{"message":"bad request: {SECRET}"}}}}"#), + headers: Box::default(), }, }; diff --git a/crates/protocol/src/client.rs b/crates/protocol/src/client.rs index 8d312e72d..720e94444 100644 --- a/crates/protocol/src/client.rs +++ b/crates/protocol/src/client.rs @@ -99,6 +99,8 @@ pub enum LlmClientError { status: http::StatusCode, /// Raw upstream error body. body: String, + /// Safe retry and correlation headers from the final failed attempt. + headers: Box, }, /// The upstream returned a response the client could not decode. diff --git a/crates/protocol/src/stream.rs b/crates/protocol/src/stream.rs index c59a66f77..f9123f663 100644 --- a/crates/protocol/src/stream.rs +++ b/crates/protocol/src/stream.rs @@ -275,6 +275,7 @@ impl AggLlmResponse { } } +/// Accumulate a chunk unless it reports a provider or decoding failure. fn push_checked_chunk( accumulator: &mut ResponseAccumulator, chunk: LlmResponseChunk, @@ -286,6 +287,7 @@ fn push_checked_chunk( LlmResponseChunk::StreamError { message } => Err(LlmClientError::UpstreamHttp { status: MID_STREAM_UPSTREAM_STATUS, body: message, + headers: Box::default(), }), chunk => { accumulator.push(chunk); diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index 95794f0d8..ccdae6b80 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -605,6 +605,7 @@ fn normalized_stream_error(chunks: &[LlmResponseChunk]) -> Option Some(LlmClientError::UpstreamHttp { status: StatusCode::BAD_GATEWAY, body: message.clone(), + headers: Box::default(), }), _ => None, }) diff --git a/crates/switchyard-runner/src/failure.rs b/crates/switchyard-runner/src/failure.rs index e73b05a29..3b4f78fba 100644 --- a/crates/switchyard-runner/src/failure.rs +++ b/crates/switchyard-runner/src/failure.rs @@ -188,6 +188,7 @@ mod tests { source: LlmClientError::UpstreamHttp { status: reqwest::StatusCode::SERVICE_UNAVAILABLE, body: format!("upstream response: {SECRET}"), + headers: Box::default(), }, }); diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index 0f1738936..00a531e1c 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -87,6 +87,15 @@ fn should_forward_upstream_header(name: &HeaderName) -> bool { .iter() .any(|prefix| name.starts_with(prefix)) } + +/// Whether an error header is useful for caller backoff or support tracing. +fn should_forward_upstream_error_header(name: &HeaderName) -> bool { + let name = name.as_str(); + matches!(name, "retry-after" | "request-id" | "x-request-id") + || name.starts_with("x-ratelimit-") + || name.starts_with("anthropic-ratelimit-") +} + /// Non-standard status used only in logs and metrics for a request whose /// downstream client disconnected before any response was written. const CLIENT_CLOSED_REQUEST: u16 = 499; @@ -665,6 +674,7 @@ fn fallback_url(base_url: &str, path_and_query: &str) -> Result Response { } } +/// Map a client failure to an HTTP response with safe upstream metadata. fn client_error(error: &LlmClientError) -> Response { match error { LlmClientError::ResponseStateLimitExceeded { .. } => error_response( @@ -1313,7 +1324,11 @@ fn client_error(error: &LlmClientError) -> Response { "upstream_error", "temporarily_unavailable", ), - LlmClientError::UpstreamHttp { status, body } => upstream_error(*status, body), + LlmClientError::UpstreamHttp { + status, + body, + headers, + } => upstream_error(*status, body, headers), LlmClientError::Transport { source } | LlmClientError::InvalidResponse { source } => { error_response( StatusCode::BAD_GATEWAY, @@ -1339,8 +1354,8 @@ fn client_error(error: &LlmClientError) -> Response { } } -// Keep the provider's message and nonempty string code in our error JSON. -fn upstream_error(status: StatusCode, body: &str) -> Response { +/// Preserve the provider's error message and safe final-attempt headers. +fn upstream_error(status: StatusCode, body: &str, headers: &HeaderMap) -> Response { let parsed = serde_json::from_str::(body).unwrap_or_default(); let error = &parsed["error"]; let message = error["message"].as_str().unwrap_or(body); @@ -1349,6 +1364,14 @@ fn upstream_error(status: StatusCode, body: &str) -> Response { .filter(|code| !code.is_empty()) .unwrap_or("upstream_error"); let mut response = error_response(status, message, "upstream_error", code); + // Other client implementations can supply Connection-nominated headers. + let mut headers = headers.clone(); + strip_hop_by_hop_headers(&mut headers); + for (name, value) in &headers { + if should_forward_upstream_error_header(name) { + response.headers_mut().append(name.clone(), value.clone()); + } + } // Provider messages and codes can quote request content; log only fixed metadata. response .extensions_mut() @@ -1406,12 +1429,19 @@ impl ApiError { } } +/// Render an API error in the caller's format without losing safe error headers. fn render_error_response(mut response: Response, wire_format: WireFormat) -> Response { let Some(error) = response.extensions_mut().remove::() else { return response; }; let log_error = response.extensions_mut().remove::(); let mut rendered = error.into_response(wire_format); + // Format-specific error rendering replaces the response, including its headers. + for (name, value) in response.headers() { + if should_forward_upstream_error_header(name) { + rendered.headers_mut().append(name.clone(), value.clone()); + } + } if let Some(log_error) = log_error { rendered.extensions_mut().insert(log_error); } @@ -2014,6 +2044,7 @@ mod tests { body: format!( r#"{{"error":{{"message":"validation failed: {LEAKED}","code":"invalid_request_{LEAKED}"}}}}"# ), + headers: Box::default(), }; for wire_format in [ WireFormat::OpenAiChat, @@ -2034,6 +2065,44 @@ mod tests { } } + /// A custom client cannot forward headers nominated as hop-by-hop by Connection. + #[test] + fn upstream_error_strips_connection_nominated_headers() { + let mut headers = HeaderMap::new(); + headers.insert( + "connection", + "x-request-id, x-ratelimit-reset-requests" + .parse() + .expect("header value"), + ); + headers.insert("x-request-id", "hop-id".parse().expect("header value")); + headers.insert( + "x-ratelimit-reset-requests", + "15s".parse().expect("header value"), + ); + headers.insert("retry-after", "15".parse().expect("header value")); + let error = LlmClientError::UpstreamHttp { + status: StatusCode::TOO_MANY_REQUESTS, + body: "rate limited".to_string(), + headers: Box::new(headers), + }; + + for wire_format in [WireFormat::OpenAiChat, WireFormat::AnthropicMessages] { + let response = render_error_response(client_error(&error), wire_format); + assert_eq!(response.status(), StatusCode::TOO_MANY_REQUESTS); + assert!(!response.headers().contains_key("x-request-id")); + assert!( + !response + .headers() + .contains_key("x-ratelimit-reset-requests") + ); + assert_eq!( + response.headers().get("retry-after"), + Some(&"15".parse().expect("header value")) + ); + } + } + // LiteLLM's cost header passes through to the client; auth headers do not. #[test] fn upstream_header_forwarding_covers_litellm_cost() { diff --git a/crates/switchyard-server/src/sse.rs b/crates/switchyard-server/src/sse.rs index 942b0ee07..8ffddab1c 100644 --- a/crates/switchyard-server/src/sse.rs +++ b/crates/switchyard-server/src/sse.rs @@ -236,6 +236,7 @@ mod tests { LlmClientError::UpstreamHttp { status: axum::http::StatusCode::BAD_GATEWAY, body: format!("upstream failed: {LEAKED}"), + headers: Box::default(), }, ))])) })?; diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 42e6a6911..ccdaea923 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -118,12 +118,66 @@ fn has_system_prompt(call: &Value, expected: &str) -> bool { }) } +/// Build a mock upstream error containing both safe and sensitive headers. +fn upstream_error_with_headers(status: StatusCode, request_id: &'static str) -> HttpResponse { + let code = if status == StatusCode::TOO_MANY_REQUESTS { + "rate_limit_exceeded" + } else { + "upstream_error" + }; + let mut response = ( + status, + Json(json!({"error": {"message": "upstream unavailable", "code": code}})), + ) + .into_response(); + let headers = response.headers_mut(); + headers.insert("retry-after", HeaderValue::from_static("15")); + headers.insert( + "x-ratelimit-reset-requests", + HeaderValue::from_static("15s"), + ); + headers.insert("x-request-id", HeaderValue::from_static(request_id)); + headers.insert("request-id", HeaderValue::from_static(request_id)); + headers.insert( + "anthropic-ratelimit-requests-reset", + HeaderValue::from_static("15s"), + ); + headers.insert("set-cookie", HeaderValue::from_static("session=secret")); + headers.insert("authorization", HeaderValue::from_static("Bearer secret")); + headers.insert("x-upstream-debug", HeaderValue::from_static("private")); + response +} + +/// Simulate chat replies and terminal failures for routing tests. async fn upstream_chat( State(calls): State>>>, Json(body): Json, ) -> HttpResponse { calls.lock().await.push(body.clone()); let prompt = user_prompt(&body); + if prompt == "error-headers-429" { + return upstream_error_with_headers(StatusCode::TOO_MANY_REQUESTS, "req-429"); + } + if prompt == "error-headers-503" { + return upstream_error_with_headers(StatusCode::SERVICE_UNAVAILABLE, "req-503"); + } + if prompt == "error-headers-connection" { + let mut response = upstream_error_with_headers(StatusCode::TOO_MANY_REQUESTS, "hop-id"); + response + .headers_mut() + .insert("connection", HeaderValue::from_static("x-request-id")); + return response; + } + if prompt == "error-headers-fallback" && body["model"] == "model/weak" { + return upstream_error_with_headers(StatusCode::TOO_MANY_REQUESTS, "req-weak"); + } + if prompt == "error-headers-final-fallback" { + return if body["model"] == "model/weak" { + upstream_error_with_headers(StatusCode::TOO_MANY_REQUESTS, "req-weak") + } else { + upstream_error_with_headers(StatusCode::SERVICE_UNAVAILABLE, "req-strong") + }; + } if prompt == "fail" { return ( StatusCode::IM_A_TEAPOT, @@ -443,6 +497,9 @@ async fn upstream_messages_requires_forwarded_oauth( Json(body): Json, ) -> HttpResponse { calls.lock().await.push(body.clone()); + if body.to_string().contains("error-headers-429") { + return upstream_error_with_headers(StatusCode::TOO_MANY_REQUESTS, "req-anthropic-429"); + } let has_expected_headers = headers .get("authorization") .and_then(|value| value.to_str().ok()) @@ -5253,3 +5310,181 @@ async fn upstream_headers_forward_on_streaming_responses() -> TestResult { ); Ok(()) } + +/// Terminal upstream errors retain only safe retry and correlation headers in both API formats. +#[tokio::test] +async fn terminal_upstream_errors_forward_safe_headers() -> TestResult { + let (upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?; + let control = send( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": ROUTE_MODEL, + "messages": [{"role": "user", "content": "upstream-headers"}] + })), + ) + .await?; + assert_eq!(control.status, StatusCode::OK); + assert_eq!( + control.headers.get("x-request-id"), + Some(&HeaderValue::from_static("req-42")) + ); + + for (prompt, status, request_id) in [ + ( + "error-headers-429", + StatusCode::TOO_MANY_REQUESTS, + "req-429", + ), + ( + "error-headers-503", + StatusCode::SERVICE_UNAVAILABLE, + "req-503", + ), + ] { + let response = send( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": ROUTE_MODEL, + "messages": [{"role": "user", "content": prompt}] + })), + ) + .await?; + assert_eq!(response.status, status); + assert_eq!( + response.headers.get("retry-after"), + Some(&HeaderValue::from_static("15")) + ); + assert_eq!( + response.headers.get("x-ratelimit-reset-requests"), + Some(&HeaderValue::from_static("15s")) + ); + assert_eq!( + response.headers.get("x-request-id"), + Some(&HeaderValue::from_static(request_id)) + ); + assert!(!response.headers.contains_key("set-cookie")); + assert!(!response.headers.contains_key("authorization")); + assert!(!response.headers.contains_key("x-upstream-debug")); + assert_eq!(response.json()?["error"]["message"], "upstream unavailable"); + } + + let nominated = send( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": ROUTE_MODEL, + "messages": [{"role": "user", "content": "error-headers-connection"}] + })), + ) + .await?; + assert_eq!(nominated.status, StatusCode::TOO_MANY_REQUESTS); + assert!(!nominated.headers.contains_key("x-request-id")); + assert_eq!( + nominated.headers.get("request-id"), + Some(&HeaderValue::from_static("hop-id")) + ); + assert_eq!( + nominated.headers.get("retry-after"), + Some(&HeaderValue::from_static("15")) + ); + + let anthropic = build_switchyard_router(load_test_config(&format!( + r#" +schema_version = 1 +[llm_clients.anthropic] +format = "anthropic_messages" +base_url = "{}" +max_retries = 0 +failure_cooldown_ms = 0 +[targets.anthropic] +id = "model/anthropic" +llm_client = "anthropic" +[routes.anthropic] +id = "route/anthropic" +type = "passthrough" +target = "anthropic" +"#, + upstream.base_url + ))?); + let response = send( + &anthropic, + "POST", + "/v1/messages", + Some(json!({ + "model": "route/anthropic", + "max_tokens": 16, + "messages": [{"role": "user", "content": "error-headers-429"}] + })), + ) + .await?; + assert_eq!(response.status, StatusCode::TOO_MANY_REQUESTS); + assert_eq!( + response.headers.get("retry-after"), + Some(&HeaderValue::from_static("15")) + ); + assert_eq!( + response.headers.get("x-request-id"), + Some(&HeaderValue::from_static("req-anthropic-429")) + ); + assert_eq!( + response.headers.get("request-id"), + Some(&HeaderValue::from_static("req-anthropic-429")) + ); + assert_eq!( + response.headers.get("anthropic-ratelimit-requests-reset"), + Some(&HeaderValue::from_static("15s")) + ); + assert_eq!(response.json()?["error"]["type"], "rate_limit_error"); + Ok(()) +} + +/// Retry guidance from an earlier candidate must not replace the final candidate's headers. +#[tokio::test] +async fn terminal_error_headers_follow_final_fallback_candidate() -> TestResult { + let upstream = MockUpstream::start().await?; + let app = build_switchyard_router(weighted_random_state(&upstream.base_url, [1000, 1])?); + + let response = send( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": ROUTE_MODEL, + "messages": [{"role": "user", "content": "error-headers-fallback"}] + })), + ) + .await?; + assert_eq!(response.status, StatusCode::OK); + assert_eq!(upstream.models().await, ["model/weak", "model/strong"]); + assert!(!response.headers.contains_key("retry-after")); + assert!(!response.headers.contains_key("x-request-id")); + + upstream.calls.lock().await.clear(); + let response = send( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": ROUTE_MODEL, + "messages": [{"role": "user", "content": "error-headers-final-fallback"}] + })), + ) + .await?; + assert_eq!(response.status, StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(upstream.models().await, ["model/weak", "model/strong"]); + assert_eq!( + response.headers.get("x-request-id"), + Some(&HeaderValue::from_static("req-strong")) + ); + assert_eq!( + response.headers.get("retry-after"), + Some(&HeaderValue::from_static("15")) + ); + assert!(!response.headers.contains_key("set-cookie")); + Ok(()) +}