Disallow new requests after ≥10 and ≥1% undocumented error responses

docs/ref/api.rst

@@ -40,5 +40,7 @@ Errors
 .. autoexception:: RequestError
     :members:
 
+.. autoexception:: TooManyUndocumentedErrors
+
 .. autoclass:: ParsedError
     :members:

docs/use/api.rst

@@ -198,5 +198,22 @@ attempt to fail. Unsuccessful responses trigger a :exc:`RequestError` and
 network errors trigger :ref:`aiohttp exceptions <aiohttp-client-reference>`.
 Other exceptions could be raised; for example, from a custom retry policy.
 
+.. _zapi-undocumented-error-limit:
+
+Undocumented error limit
+========================
+
+.. versionadded:: VERSION
+
+Error responses with an HTTP status code in the 500-599 range (503, 520 and
+521 excluded) are usually a sign of an issue on the Zyte API side. Once a
+client has received 10 or more of them, and they account for 1% or more of the
+requests it has sent, that client stops sending requests: every call raises
+:exc:`~zyte_api.TooManyUndocumentedErrors` instead.
+
+The limit is per client object, so a new client can be used to resume sending
+requests. However, check https://status.zyte.com/ or contact `support
+<https://support.zyte.com/support/tickets/new>`_ first.
+
 
 .. seealso:: :ref:`api-ref`

tests/test_async.py

@@ -7,7 +7,12 @@
 
 import pytest
 
-from zyte_api import AggressiveRetryFactory, AsyncZyteAPI, RequestError
+from zyte_api import (
+    AggressiveRetryFactory,
+    AsyncZyteAPI,
+    RequestError,
+    TooManyUndocumentedErrors,
+)
 from zyte_api._utils import create_session
 from zyte_api.aio.client import AsyncClient
 from zyte_api.apikey import NoApiKey
@@ -366,3 +371,53 @@ def test_retrying_class():
     AsyncRetrying subclass or similar instead of an instance of it."""
     with pytest.raises(ValueError, match="must be an instance of AsyncRetrying"):
         AsyncZyteAPI(api_key="foo", retrying=AggressiveRetryFactory)  # type: ignore[arg-type]
+
+
+@pytest.mark.parametrize(
+    ("status_codes", "n_attempts", "raises"),
+    (
+        # Undocumented error responses must be at least 10 and at least 1% of
+        # all attempts.
+        ({500: 9}, 9, False),
+        ({500: 10}, 10, True),
+        ({500: 10}, 1001, False),  # 0.999…%
+        ({500: 10}, 1000, True),  # 1%
+        # Rate-limiting and download errors do not count.
+        ({503: 10, 520: 10, 521: 10}, 30, False),
+    ),
+)
+@pytest.mark.asyncio
+async def test_too_many_undocumented_errors(
+    status_codes, n_attempts, raises, mockserver
+):
+    client = AsyncZyteAPI(api_key="a", api_url=mockserver.urljoin("/"))
+    client.agg_stats.status_codes.update(status_codes)
+    client.agg_stats.n_attempts = n_attempts
+    query = {"url": "https://a.example", "httpResponseBody": True}
+    if raises:
+        with pytest.raises(TooManyUndocumentedErrors):
+            await client.get(query)
+    else:
+        await client.get(query)
+
+
+@pytest.mark.asyncio
+async def test_too_many_undocumented_errors_e2e(mockserver):
+    client = AsyncZyteAPI(api_key="a", api_url=mockserver.urljoin("/"))
+    error_query = {"url": "https://e500.example", "httpResponseBody": True}
+    for _ in range(10):
+        with pytest.raises(RequestError):
+            await client.get(error_query, handle_retries=False)
+
+    # Requests are disallowed regardless of the query, and the same exception
+    # is reused for every subsequent call.
+    good_query = {"url": "https://a.example", "httpResponseBody": True}
+    with pytest.raises(TooManyUndocumentedErrors) as first:
+        await client.get(good_query)
+    with pytest.raises(TooManyUndocumentedErrors) as second:
+        await client.get(good_query)
+    assert first.value is second.value
+
+    # A separate client is unaffected.
+    other_client = AsyncZyteAPI(api_key="a", api_url=mockserver.urljoin("/"))
+    await other_client.get(good_query)

zyte_api/__init__.py

@@ -3,7 +3,7 @@
 """
 
 from ._async import AsyncZyteAPI, AuthInfo
-from ._errors import RequestError
+from ._errors import RequestError, TooManyUndocumentedErrors
 from ._retry import (
     AggressiveRetryFactory,
     RetryFactory,
@@ -31,6 +31,7 @@
     "ParsedError",
     "RequestError",
     "RetryFactory",
+    "TooManyUndocumentedErrors",
     "ZyteAPI",
     "aggressive_retrying",
     "stop_after_uninterrupted_delay",

zyte_api/_async.py

@@ -13,7 +13,7 @@
 from zyte_api._x402 import _x402Handler
 from zyte_api.apikey import NoApiKey, read_dotenv_auth
 
-from ._errors import RequestError
+from ._errors import RequestError, TooManyUndocumentedErrors
 from ._retry import zyte_api_retrying
 from ._utils import _AIO_API_TIMEOUT, create_session
 from .constants import API_URL
@@ -33,6 +33,11 @@
 
     _ResponseFuture = Awaitable[dict[str, Any]]
 
+# Undocumented error responses must reach both thresholds, in absolute number
+# and as a share of all requests sent, before a client stops sending requests.
+_UNDOCUMENTED_ERROR_MIN = 10
+_UNDOCUMENTED_ERROR_RATIO = 0.01
+
 
 def _post_func(
     session: aiohttp.ClientSession | None,
@@ -140,6 +145,7 @@ def __init__(
         self.user_agent = user_agent or USER_AGENT
         self.trust_env = trust_env
         self._semaphore = asyncio.Semaphore(n_conn)
+        self._too_many_undocumented_errors: TooManyUndocumentedErrors | None = None
         self._auth: str | _x402Handler
         self.auth: AuthInfo
         self.api_url: str
@@ -196,6 +202,19 @@ def api_key(self) -> str:
             "api_key is not available when using an Ethereum private key, use auth.key instead."
         )
 
+    def _check_undocumented_errors(self) -> None:
+        if self._too_many_undocumented_errors is not None:
+            raise self._too_many_undocumented_errors
+        errors = self.agg_stats._n_undocumented_errors
+        if (
+            errors >= _UNDOCUMENTED_ERROR_MIN
+            and errors >= _UNDOCUMENTED_ERROR_RATIO * self.agg_stats.n_attempts
+        ):
+            self._too_many_undocumented_errors = TooManyUndocumentedErrors(
+                errors, self.agg_stats.n_attempts
+            )
+            raise self._too_many_undocumented_errors
+
     async def get(
         self,
         query: dict[str, Any],
@@ -206,6 +225,8 @@ async def get(
         retrying: AsyncRetrying | None = None,
     ) -> dict[str, Any]:
         """Asynchronous equivalent to :meth:`ZyteAPI.get`."""
+        self._check_undocumented_errors()
+
         retrying = retrying or self.retrying
         owned_session: aiohttp.ClientSession | None = None
         if session is None:

zyte_api/_errors.py

@@ -10,6 +10,29 @@
 logger = logging.getLogger("zyte_api")
 
 
+def _is_undocumented_status(status: int) -> bool:
+    # 503 is rate limiting, 520 and 521 are download errors.
+    return status >= 500 and status not in {503, 520, 521}
+
+
+class TooManyUndocumentedErrors(RuntimeError):
+    """Exception raised by a client that has received :ref:`too many
+    undocumented error responses <zapi-undocumented-error-limit>`.
+
+    .. versionadded:: VERSION
+    """
+
+    def __init__(self, errors: int, total: int):
+        super().__init__(
+            f"Too many undocumented error responses received from Zyte API "
+            f"({errors} out of {total}, {errors / total:.2%}). This client "
+            f"will not send any more Zyte API requests. Please, check "
+            f"https://status.zyte.com/ or contact support "
+            f"(https://support.zyte.com/support/tickets/new) before sending "
+            f"more requests like the ones causing these error responses."
+        )
+
+
 class RequestError(ClientResponseError):
     """Exception raised upon receiving a :ref:`rate-limiting
     <zapi-rate-limit>` or :ref:`unsuccessful

zyte_api/_retry.py

@@ -25,7 +25,7 @@
 )
 from tenacity.stop import stop_base, stop_never
 
-from ._errors import RequestError
+from ._errors import RequestError, _is_undocumented_status
 
 if TYPE_CHECKING:
     from collections.abc import Callable
@@ -158,11 +158,7 @@ def _download_error(exc: BaseException) -> bool:
 
 
 def _undocumented_error(exc: BaseException) -> bool:
-    return (
-        isinstance(exc, RequestError)
-        and exc.status >= 500
-        and exc.status not in {503, 520, 521}
-    )
+    return isinstance(exc, RequestError) and _is_undocumented_status(exc.status)
 
 
 def _402_error(exc: BaseException) -> bool:

zyte_api/stats.py

@@ -8,6 +8,7 @@
 import attr
 from runstats import Statistics
 
+from zyte_api._errors import _is_undocumented_status
 from zyte_api.errors import ParsedError
 
 if TYPE_CHECKING:
@@ -92,6 +93,14 @@ def n_processed(self) -> float:
         """Total number of processed URLs"""
         return self.n_success + self.n_fatal_errors
 
+    @property
+    def _n_undocumented_errors(self) -> int:
+        return sum(
+            count
+            for status, count in self.status_codes.items()
+            if _is_undocumented_status(status)
+        )
+
 
 @attr.s
 class ResponseStats: