Add opt-in crawl_timeout and shield page cleanup from cancellation

crawl4ai/async_configs.py

@@ -266,7 +266,7 @@ class UntrustedConfigError(ValueError):
         "no_cache_write", "check_cache_freshness", "cache_validation_timeout",
         "fetch_ssl_certificate",
         # timing / waiting
-        "wait_until", "page_timeout", "wait_for", "wait_for_timeout",
+        "wait_until", "page_timeout", "crawl_timeout", "wait_for", "wait_for_timeout",
         "body_visibility_timeout",
         "wait_for_images", "delay_before_return_html", "mean_delay", "max_range",
         # scrolling / rendering
@@ -325,7 +325,7 @@ def _cap_timeout(v):
         return min(int(v), _MAX_TIMEOUT_MS)
 
     if type_name == "CrawlerRunConfig":
-        for f in ("page_timeout", "wait_for_timeout", "body_visibility_timeout"):
+        for f in ("page_timeout", "crawl_timeout", "wait_for_timeout", "body_visibility_timeout"):
             if f in params:
                 params[f] = _cap_timeout(params[f])
         if isinstance(params.get("max_scroll_steps"), int):
@@ -1473,6 +1473,9 @@ class CrawlerRunConfig():
                           Default: "domcontentloaded".
         page_timeout (int): Timeout in ms for page operations like navigation.
                             Default: 60000 (60 seconds).
+        crawl_timeout (int or None): Timeout in ms for the whole page visit, from navigation to final HTML,
+                                     including js_code and hooks. None = no limit.
+                                     Default: None.
         wait_for (str or None): A CSS selector or JS condition to wait for before extracting content.
                                 Default: None.
         wait_for_timeout (int or None): Specific timeout in ms for the wait_for condition.
@@ -1666,6 +1669,7 @@ def __init__(
         # Page Navigation and Timing Parameters
         wait_until: str = "domcontentloaded",
         page_timeout: int = PAGE_TIMEOUT,
+        crawl_timeout: Optional[int] = None,
         wait_for: str = None,
         wait_for_timeout: int = None,
         wait_for_images: bool = False,
@@ -1796,6 +1800,7 @@ def __init__(
         # Page Navigation and Timing Parameters
         self.wait_until = wait_until
         self.page_timeout = page_timeout
+        self.crawl_timeout = crawl_timeout
         self.wait_for = wait_for
         self.wait_for_timeout = wait_for_timeout
         self.wait_for_images = wait_for_images
@@ -2173,6 +2178,7 @@ def to_dict(self):
             "shared_data": self.shared_data,
             "wait_until": self.wait_until,
             "page_timeout": self.page_timeout,
+            "crawl_timeout": self.crawl_timeout,
             "wait_for": self.wait_for,
             "wait_for_timeout": self.wait_for_timeout,
             "wait_for_images": self.wait_for_images,

crawl4ai/async_crawler_strategy.py

@@ -526,18 +526,9 @@ async def _crawl_web(
             AsyncCrawlResponse: The response containing HTML, headers, status code, and optional data
         """
         config.url = url
-        response_headers = {}
-        execution_result = None
-        status_code = None
-        redirected_url = url
-        redirected_status_code = None
 
         # Reset downloaded files list for new crawl
         self._downloaded_files = []
-
-        # Initialize capture lists
-        captured_requests = []
-        captured_console = []
 
         # Handle user agent with magic mode.
         # For persistent contexts the UA is locked at browser launch time
@@ -570,10 +561,32 @@ async def _crawl_web(
         # previous navigation to prevent timeouts on the next goto().
         if config.session_id:
             try:
-                await page.evaluate("window.stop()")
+                await asyncio.wait_for(page.evaluate("window.stop()"), 2)  # a session page may already be hung
             except Exception:
                 pass
 
+        if not config.crawl_timeout:
+            return await self._crawl_page(url, config, page, context, ua_changed)
+        try:
+            return await asyncio.wait_for(
+                asyncio.create_task(self._crawl_page(url, config, page, context, ua_changed)), config.crawl_timeout / 1000
+            )
+        except asyncio.TimeoutError:
+            await self._close_unresponsive_page(page, config)
+            raise RuntimeError(f"Crawl exceeded crawl_timeout of {config.crawl_timeout} ms")
+
+    async def _crawl_page(
+        self, url: str, config: CrawlerRunConfig, page: Page, context, ua_changed: bool
+    ) -> AsyncCrawlResponse:
+        """The page visit itself (navigation to final HTML plus cleanup); bounded by crawl_timeout in _crawl_web."""
+        response_headers = {}
+        execution_result = None
+        status_code = None
+        redirected_url = url
+        redirected_status_code = None
+        captured_requests = []
+        captured_console = []
+
         try:
             # Push updated UA + sec-ch-ua to the page so the server sees them
             if ua_changed:
@@ -1198,37 +1211,77 @@ async def get_delayed_content(delay: float = 5.0) -> str:
             raise e
 
         finally:
-            # Always clean up event listeners to prevent accumulation
-            # across reuses (even for session pages).
-            try:
-                if config.capture_network_requests:
-                    page.remove_listener("request", handle_request_capture)
-                    page.remove_listener("response", handle_response_capture)
-                    page.remove_listener("requestfailed", handle_request_failed_capture)
-                if config.capture_console_messages:
-                    if hasattr(self.adapter, 'retrieve_console_messages'):
-                        final_messages = await self.adapter.retrieve_console_messages(page)
-                        captured_console.extend(final_messages)
-                    await self.adapter.cleanup_console_capture(page, handle_console, handle_error)
-            except Exception:
-                pass
-
-            if not config.session_id:
-                # ALWAYS decrement refcount first — must succeed even if
-                # the browser crashed or the page is in a bad state.
+            async def _cleanup():
+                # Always clean up event listeners to prevent accumulation
+                # across reuses (even for session pages).
                 try:
-                    await self.browser_manager.release_page_with_context(page)
+                    if config.capture_network_requests:
+                        page.remove_listener("request", handle_request_capture)
+                        page.remove_listener("response", handle_response_capture)
+                        page.remove_listener("requestfailed", handle_request_failed_capture)
+                    if config.capture_console_messages:
+                        if hasattr(self.adapter, 'retrieve_console_messages'):
+                            final_messages = await asyncio.wait_for(self.adapter.retrieve_console_messages(page), 5)
+                            captured_console.extend(final_messages)
+                        await asyncio.wait_for(self.adapter.cleanup_console_capture(page, handle_console, handle_error), 5)
                 except Exception:
                     pass
 
-                # Close the page unless it's the last one in a headless/managed browser
-                try:
-                    all_contexts = page.context.browser.contexts
-                    total_pages = sum(len(context.pages) for context in all_contexts)
-                    if not (total_pages <= 1 and (self.browser_config.use_managed_browser or self.browser_config.headless)):
-                        await page.close()
-                except Exception:
-                    pass
+                if not config.session_id:
+                    # ALWAYS decrement refcount first — must succeed even if
+                    # the browser crashed or the page is in a bad state.
+                    try:
+                        await self.browser_manager.release_page_with_context(page)
+                    except Exception:
+                        pass
+
+                    # Close the page unless it's the last one in a headless/managed browser
+                    try:
+                        all_contexts = page.context.browser.contexts
+                        total_pages = sum(len(context.pages) for context in all_contexts)
+                        if not (total_pages <= 1 and (self.browser_config.use_managed_browser or self.browser_config.headless)):
+                            await page.close()
+                    except Exception:
+                        pass
+
+            # Shielded so a cancel landing mid-cleanup cannot skip page.close(); the cancel is re-raised once cleanup is done
+            cleanup = asyncio.create_task(_cleanup())
+            try:
+                await asyncio.shield(cleanup)
+            except asyncio.CancelledError:
+                await asyncio.shield(cleanup)
+                raise
+
+    async def _close_unresponsive_page(self, page: Page, config: CrawlerRunConfig) -> None:
+        """
+        Close (or drop the session of) a page whose crawl hit crawl_timeout.
+
+        Args:
+            page (Page): The Playwright page instance
+            config (CrawlerRunConfig): Crawler Config to check for session_id
+        """
+        async def _close():
+            browser = page.context.browser
+            if not page.is_closed() and browser and sum(len(c.pages) for c in browser.contexts) <= 1:
+                await page.context.new_page()  # a headed managed Chrome exits when its last tab closes
+            if config.session_id:
+                self.logger.warning(
+                    message="Dropping session {session_id}: crawl exceeded crawl_timeout",
+                    tag="TIMEOUT",
+                    params={"session_id": config.session_id},
+                )
+                await self.browser_manager.kill_session(config.session_id)
+            elif not page.is_closed():
+                await page.close()
+
+        try:
+            await asyncio.wait_for(_close(), 5)
+        except Exception as e:
+            self.logger.warning(
+                message="Could not close unresponsive page: {error}",
+                tag="TIMEOUT",
+                params={"error": str(e)},
+            )
 
     # async def _handle_full_page_scan(self, page: Page, scroll_delay: float = 0.1):
     async def _handle_full_page_scan(self, page: Page, scroll_delay: float = 0.1, max_scroll_steps: Optional[int] = None):

deploy/docker/api.py

@@ -126,6 +126,15 @@ async def hset_with_ttl(redis, key: str, mapping: dict, config: dict):
         await redis.expire(key, ttl)
 
 
+def apply_base_config(cfg: CrawlerRunConfig, config: dict, keys=None) -> CrawlerRunConfig:
+    """Fill fields the request left unset from config.yml crawler.base_config (all of them, or just `keys`)."""
+    base = config["crawler"]["base_config"]
+    for key in keys or base:
+        if hasattr(cfg, key) and getattr(cfg, key) in (None, ""):
+            setattr(cfg, key, base[key])
+    return cfg
+
+
 async def handle_llm_qa(
     url: str,
     query: str,
@@ -156,7 +165,7 @@ async def handle_llm_qa(
         from egress_broker import enforce_egress
         enforce_egress(browser_cfg)
         crawler = await get_crawler(browser_cfg)
-        result = await crawler.arun(url)
+        result = await crawler.arun(url, config=apply_base_config(CrawlerRunConfig(), cfg, keys=("crawl_timeout",)))
         _raise_for_crawl_failure(result)
         content = result.markdown.fit_markdown or result.markdown.raw_markdown
 
@@ -266,11 +275,11 @@ async def process_llm_extraction(
         async with AsyncWebCrawler(config=worker_browser_cfg) as crawler:
             result = await crawler.arun(
                 url=url,
-                config=CrawlerRunConfig(
+                config=apply_base_config(CrawlerRunConfig(
                     extraction_strategy=llm_strategy,
                     scraping_strategy=LXMLWebScrapingStrategy(),
                     cache_mode=cache_mode
-                )
+                ), config, keys=("crawl_timeout",))
             )
 
         if not result.success:
@@ -390,11 +399,11 @@ async def handle_markdown_request(
         crawler = await get_crawler(browser_cfg)
         result = await crawler.arun(
             url=decoded_url,
-            config=CrawlerRunConfig(
+            config=apply_base_config(CrawlerRunConfig(
                 markdown_generator=md_generator,
                 scraping_strategy=LXMLWebScrapingStrategy(),
                 cache_mode=cache_mode
-            )
+            ), config, keys=("crawl_timeout",))
         )
 
         _raise_for_crawl_failure(result)
@@ -719,29 +728,19 @@ async def handle_crawl_request(
             hooks_status = _attach_declarative_hooks(crawler, hooks_config)
             logger.info(f"Hooks attachment status: {hooks_status['status']}")
 
-        base_config = config["crawler"]["base_config"]
-
         # Build the config(s) to pass to arun/arun_many
         if crawler_configs and len(urls) > 1:
             # Per-URL config list: deserialize each and apply base_config
             config_list = [CrawlerRunConfig.load(cc, provenance=Provenance.UNTRUSTED) for cc in crawler_configs]
             for cfg in config_list:
-                for key, value in base_config.items():
-                    if hasattr(cfg, key):
-                        current_value = getattr(cfg, key)
-                        if current_value is None or current_value == "":
-                            setattr(cfg, key, value)
+                apply_base_config(cfg, config)
                 # SSRF: per-URL PDF strategies need the validator wired too
                 if isinstance(cfg.scraping_strategy, PDFContentScrapingStrategy):
                     cfg.scraping_strategy.url_validator = validate_url_destination
             effective_config = config_list
         else:
             # Single config (original behavior)
-            for key, value in base_config.items():
-                if hasattr(crawler_config, key):
-                    current_value = getattr(crawler_config, key)
-                    if current_value is None or current_value == "":
-                        setattr(crawler_config, key, value)
+            apply_base_config(crawler_config, config)
             effective_config = crawler_config
 
         results = []
@@ -911,6 +910,7 @@ async def handle_stream_crawl_request(
         crawler_config = CrawlerRunConfig.load(
             crawler_config, provenance=Provenance.UNTRUSTED
         )
+        apply_base_config(crawler_config, config)
         from governor import clamp_deep_crawl
 
         clamp_deep_crawl(crawler_config)

deploy/docker/config.yml

@@ -74,6 +74,7 @@ security:
 crawler:
   base_config:
     simulate_user: true
+    crawl_timeout: 180000   # ms; bounds the whole page visit so a hung page cannot pin a renderer
   memory_threshold_percent: 95.0
   rate_limiter:
     enabled: true

deploy/docker/server.py

@@ -27,7 +27,7 @@
 from api import (
     handle_markdown_request, handle_llm_qa,
     handle_stream_crawl_request, handle_crawl_request,
-    stream_results
+    stream_results, apply_base_config
 )
 from schemas import (
     CrawlRequestWithHooks,
@@ -658,7 +658,7 @@ async def generate_html(
     Use when you need sanitized HTML structures for building schemas or further processing.
     """
     validate_url_scheme(body.url, allow_raw=True)
-    cfg = CrawlerRunConfig()
+    cfg = apply_base_config(CrawlerRunConfig(), config, keys=("crawl_timeout",))
     crawler = None
     try:
         crawler = await get_crawler(get_default_browser_config())
@@ -754,6 +754,7 @@ async def generate_screenshot(
     crawler = None
     try:
         cfg = CrawlerRunConfig(screenshot=True, screenshot_wait_for=body.screenshot_wait_for, wait_for_images=body.wait_for_images)
+        apply_base_config(cfg, config, keys=("crawl_timeout",))
         crawler = await get_crawler(get_default_browser_config())
         results = await crawler.arun(url=body.url, config=cfg)
         if not results[0].success:
@@ -794,7 +795,7 @@ async def generate_pdf(
     legacy_output_path = body.model_dump(include={"output_path"}).get("output_path")
     crawler = None
     try:
-        cfg = CrawlerRunConfig(pdf=True)
+        cfg = apply_base_config(CrawlerRunConfig(pdf=True), config, keys=("crawl_timeout",))
         crawler = await get_crawler(get_default_browser_config())
         results = await crawler.arun(url=body.url, config=cfg)
         if not results[0].success:
@@ -877,7 +878,7 @@ class MarkdownGenerationResult(BaseModel):
         raise HTTPException(400, str(e))
     crawler = None
     try:
-        cfg = CrawlerRunConfig(js_code=body.scripts)
+        cfg = apply_base_config(CrawlerRunConfig(js_code=body.scripts), config, keys=("crawl_timeout",))
         crawler = await get_crawler(get_default_browser_config())
         results = await crawler.arun(url=body.url, config=cfg)
         if not results[0].success:

docs/md_v2/api/parameters.md

@@ -140,6 +140,7 @@ Use these for controlling whether you read or write from a local content cache.
 |----------------------------|-------------------------|----------------------------------------------------------------------------------------------------------------------|
 | **`wait_until`**           | `str` (domcontentloaded)| Condition for navigation to "complete". Often `"networkidle"` or `"domcontentloaded"`.                               |
 | **`page_timeout`**         | `int` (60000 ms)        | Timeout for page navigation or JS steps. Increase for slow sites.                                                    |
+| **`crawl_timeout`**        | `int or None` (None)    | Timeout in ms for the whole page visit, from navigation to final HTML, including `js_code` and hooks. On expiry the page is closed and the crawl fails. Applies per attempt when `max_retries` or a proxy list is set. None = no limit. |
 | **`wait_for`**             | `str or None`           | Wait for a CSS (`"css:selector"`) or JS (`"js:() => bool"`) condition before content extraction.                     |
 | **`wait_for_timeout`**     | `int or None` (None)    | Specific timeout in ms for the `wait_for` condition. If None, uses `page_timeout`.                                   |
 | **`wait_for_images`**      | `bool` (False)          | Wait for images to load before finishing. Slows down if you only want text.                                          |

docs/md_v2/core/browser-crawler-config.md

@@ -298,6 +298,7 @@ class CrawlerRunConfig:
     - **`scan_full_page`**: If `True`, scroll through the entire page to load all content
     - **`wait_until`**: Condition to wait for when navigating (e.g., "domcontentloaded", "networkidle")
     - **`page_timeout`**: Timeout in milliseconds for page operations (default: 60000)
+    - **`crawl_timeout`**: Timeout in milliseconds for the whole page visit, navigation to final HTML, including `js_code` and hooks (default: None, no limit)
     - **`delay_before_return_html`**: Delay in seconds before retrieving final HTML.
 
 13.⠀**`url_matcher`** & **`match_mode`**:  

docs/md_v2/core/page-interaction.md

@@ -218,8 +218,9 @@ result = await crawler.arun(url="https://github.com/search", config=config)
 ## 4. Timing Control
 
 1. **`page_timeout`** (ms): Overall page load or script execution time limit.  
-2. **`delay_before_return_html`** (seconds): Wait an extra moment before capturing the final HTML.  
-3. **`mean_delay`** & **`max_range`**: If you call `arun_many()` with multiple URLs, these add a random pause between each request.
+2. **`crawl_timeout`** (ms): Limit for the whole page visit, navigation to final HTML, including `js_code` and hooks. Applies per attempt when `max_retries` or a proxy list is set. None = no limit.  
+3. **`delay_before_return_html`** (seconds): Wait an extra moment before capturing the final HTML.  
+4. **`mean_delay`** & **`max_range`**: If you call `arun_many()` with multiple URLs, these add a random pause between each request.
 
 **Example**: