fix(adagents): raise AdagentsAccessBlockedError on 403 + cf-mitigated: challenge (#837)

* fix(adagents): raise AdagentsAccessBlockedError on 403 + cf-mitigated: challenge

When _fetch_adagents_url receives HTTP 403 with cf-mitigated: challenge,
raise AdagentsAccessBlockedError (AdagentsValidationError subclass) so
callers can distinguish Cloudflare bot-management blocks from other errors.
Existing except AdagentsValidationError handlers catch it automatically.

Closes #801

* fix(adagents): address pre-PR review findings

- Add parametrized test for capitalized cf-mitigated: Challenge (Cloudflare's actual casing)
- Add .strip() to header comparison for OWS robustness
- Document AdagentsAccessBlockedError in fetch_adagents Raises section
- Add explanatory comments at sites that intentionally swallow the subclass

* test(adagents): avoid URL substring assertion

---------

Co-authored-by: Claude <noreply@anthropic.com>

Author: bokelley

src/adcp/__init__.py

@@ -49,6 +49,7 @@
 )
 from adcp.client import ADCPClient, ADCPMultiAgentClient, Checkpoint
 from adcp.exceptions import (  # noqa: F401
+    AdagentsAccessBlockedError,
     AdagentsNotFoundError,
     AdagentsTimeoutError,
     AdagentsValidationError,
@@ -947,6 +948,7 @@ def get_adcp_version() -> str:
     "ADCPSigningRequiredError",
     "ADCPWebhookError",
     "ADCPWebhookSignatureError",
+    "AdagentsAccessBlockedError",
     "AdagentsValidationError",
     "AdagentsNotFoundError",
     "AdagentsTimeoutError",

src/adcp/adagents.py

@@ -22,7 +22,12 @@
 import httpx
 from pydantic import Field
 
-from adcp.exceptions import AdagentsNotFoundError, AdagentsTimeoutError, AdagentsValidationError
+from adcp.exceptions import (
+    AdagentsAccessBlockedError,
+    AdagentsNotFoundError,
+    AdagentsTimeoutError,
+    AdagentsValidationError,
+)
 from adcp.types.base import AdCPBaseModel
 from adcp.validation import ValidationError, validate_adagents
 
@@ -779,6 +784,9 @@ async def fetch_adagents(
     Raises:
         AdagentsNotFoundError: If adagents.json was not found via any
             discovery path.
+        AdagentsAccessBlockedError: If the publisher's CDN returns HTTP
+            403 with ``cf-mitigated: challenge`` (Cloudflare bot-management
+            block). Subclass of ``AdagentsValidationError``.
         AdagentsValidationError: If JSON is invalid, malformed, or
             redirects exceed maximum depth or form a loop.
         AdagentsTimeoutError: If request times out.
@@ -885,6 +893,10 @@ async def _try_managerdomain_fallback(
         )
         return data
     except (AdagentsNotFoundError, AdagentsValidationError, AdagentsTimeoutError):
+        # AdagentsAccessBlockedError (a AdagentsValidationError subclass) is
+        # intentionally swallowed here: a bot-blocked manager domain is treated
+        # as "not available", and fetch_adagents re-raises the original
+        # AdagentsNotFoundError for the primary domain.
         return None
 
 
@@ -1077,6 +1089,12 @@ async def _fetch_adagents_url(
         parsed = urlparse(url)
         raise AdagentsNotFoundError(parsed.netloc)
 
+    if status_code == 403 and (
+        response_headers.get("cf-mitigated", "").strip().lower() == "challenge"
+    ):
+        parsed = urlparse(url)
+        raise AdagentsAccessBlockedError(parsed.netloc)
+
     if status_code != 200:
         raise AdagentsValidationError(f"Failed to fetch adagents.json: HTTP {status_code}")
 
@@ -1925,7 +1943,10 @@ async def fetch_authorization_for_domain(
             return (domain, AuthorizationContext(properties))
 
         except (AdagentsNotFoundError, AdagentsValidationError, AdagentsTimeoutError):
-            # Silently skip domains with missing or invalid adagents.json
+            # Silently skip domains with missing or invalid adagents.json.
+            # AdagentsAccessBlockedError (AdagentsValidationError subclass) is
+            # intentionally swallowed: a bot-blocked domain is treated as
+            # authorization-unavailable, same as a missing file.
             return (domain, None)
 
     # Fetch all domains in parallel

src/adcp/exceptions.py

@@ -285,6 +285,38 @@ def __init__(self, publisher_domain: str, timeout: float):
         super().__init__(message, None, None, suggestion)
 
 
+class AdagentsAccessBlockedError(AdagentsValidationError):
+    """adagents.json fetch blocked by publisher-side bot management (403, cf-mitigated: challenge).
+
+    Only surfaces in direct-fetch workflows (``fetch_adagents``). SDK callers that
+    use ``fetch_agent_authorizations`` avoid this entirely — the AAO directory crawler
+    handles publisher fetches and serves cached results without exposing the SDK to
+    publisher-side bot management.
+
+    If you need to catch this specifically without catching all
+    ``AdagentsValidationError``s, use ``except AdagentsAccessBlockedError``.
+    """
+
+    def __init__(self, publisher_domain: str):
+        """Initialize bot-management blocked error."""
+        self.publisher_domain = publisher_domain
+        message = (
+            f"adagents.json blocked by bot management for {publisher_domain} "
+            f"(HTTP 403, cf-mitigated: challenge)"
+        )
+        suggestion = (
+            "The publisher's origin blocked this request with a Cloudflare bot management\n"
+            "     challenge. This only affects direct adagents.json fetches (fetch_adagents).\n"
+            "\n"
+            "     To unblock local debugging:\n"
+            "     - Retry with a browser-like User-Agent via the user_agent= parameter, e.g.\n"
+            '       user_agent="Mozilla/5.0"\n'
+            "     - Or call fetch_agent_authorizations() to query the AAO directory instead,\n"
+            "       which bypasses publisher-side bot management entirely."
+        )
+        super().__init__(message, None, None, suggestion)
+
+
 class ADCPTaskError(ADCPError):
     """A task returned an ADCP error response.
 

tests/fixtures/public_api_snapshot.json

@@ -30,6 +30,7 @@
     "ActivateSignalResponse1",
     "ActivateSignalSuccessResponse",
     "AdAgentsValidationResult",
+    "AdagentsAccessBlockedError",
     "AdagentsCacheEntry",
     "AdagentsEntryError",
     "AdagentsFetchResult",

tests/test_adagents.py

@@ -23,6 +23,7 @@
     verify_agent_authorization,
 )
 from adcp.exceptions import (
+    AdagentsAccessBlockedError,
     AdagentsValidationError,
 )
 
@@ -674,6 +675,45 @@ def _stream(method, url, **kwargs):
         # Should stop after reasonable number of redirects (not go forever)
         assert call_count[0] <= 10
 
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("cf_value", ["challenge", "Challenge"])
+    async def test_fetch_403_cf_mitigated_raises_access_blocked(self, cf_value):
+        """403 + cf-mitigated: challenge raises AdagentsAccessBlockedError (case-insensitive)."""
+        from adcp.adagents import fetch_adagents
+
+        mock_response = MagicMock()
+        mock_response.status_code = 403
+        mock_response.headers = httpx.Headers({"cf-mitigated": cf_value})
+        mock_response.json.return_value = None
+
+        mock_client = create_mock_httpx_client(mock_response)
+
+        with pytest.raises(AdagentsAccessBlockedError, match="cf-mitigated: challenge"):
+            await fetch_adagents("cafemedia.com", client=mock_client)
+
+    @pytest.mark.asyncio
+    async def test_fetch_403_no_cf_header_raises_generic_validation_error(self):
+        """Plain 403 without cf-mitigated header raises generic AdagentsValidationError."""
+        from adcp.adagents import fetch_adagents
+
+        mock_response = MagicMock()
+        mock_response.status_code = 403
+        mock_response.headers = httpx.Headers({})
+        mock_response.json.return_value = None
+
+        mock_client = create_mock_httpx_client(mock_response)
+
+        with pytest.raises(AdagentsValidationError, match="HTTP 403") as exc_info:
+            await fetch_adagents("example.com", client=mock_client)
+        assert not isinstance(exc_info.value, AdagentsAccessBlockedError)
+
+    def test_access_blocked_is_subclass_of_validation_error(self):
+        """AdagentsAccessBlockedError is catchable as AdagentsValidationError."""
+        err = AdagentsAccessBlockedError("cafemedia.com")
+        assert isinstance(err, AdagentsValidationError)
+        assert "cf-mitigated: challenge" in str(err)
+        assert err.publisher_domain == "cafemedia.com"
+
 
 class TestSSRFProtection:
     """Test SSRF protections on authoritative_location redirects."""