diff --git a/crates/fetchkit/src/fetchers/default.rs b/crates/fetchkit/src/fetchers/default.rs index 9d7f7fe..1b894b3 100644 --- a/crates/fetchkit/src/fetchers/default.rs +++ b/crates/fetchkit/src/fetchers/default.rs @@ -786,6 +786,9 @@ pub(crate) async fn send_request_following_redirects( let mut redirect_chain = Vec::new(); for redirect_count in 0..=MAX_REDIRECTS { + // Every outbound request, including generated probes and redirect hops, must + // honor the caller's allow/block prefix policy. + super::validate_url_policy(¤t_url, options)?; // THREAT[TM-AUTH]: re-sign bot-auth headers per hop so each authority is covered. let request_headers = apply_bot_auth_if_enabled(headers.clone(), options, ¤t_url); // THREAT[TM-SSRF-001]/[TM-SSRF-005]: resolve-then-check produces the pinned addrs @@ -2030,6 +2033,35 @@ mod tests { assert!(content.contains("/openapi.json")); } + #[tokio::test] + async fn test_agent_resource_probes_honor_url_prefix_policy() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/docs/page")) + .respond_with( + ResponseTemplate::new(200) + .set_body_string("

Page

") + .insert_header("content-type", "text/html"), + ) + .mount(&server) + .await; + + let fetcher = DefaultFetcher::new(); + let options = FetchOptions { + enable_markdown: true, + allow_prefixes: vec![format!("{}/docs", server.uri())], + dns_policy: DnsPolicy::allow_all(), + ..Default::default() + }; + let request = FetchRequest::new(format!("{}/docs/page", server.uri())).as_markdown(); + fetcher.fetch(&request, &options).await.unwrap(); + + let requests = server.received_requests().await.unwrap(); + assert_eq!(requests.len(), 1); + assert_eq!(requests[0].method, reqwest::Method::GET); + assert_eq!(requests[0].url.path(), "/docs/page"); + } + #[tokio::test] async fn test_paywall_detection() { let server = MockServer::start().await;