diff --git a/crawl4ai/async_url_seeder.py b/crawl4ai/async_url_seeder.py index 22fa8f630..a468c6204 100644 --- a/crawl4ai/async_url_seeder.py +++ b/crawl4ai/async_url_seeder.py @@ -225,7 +225,10 @@ def _parse_head(src: str) -> Dict[str, Any]: except json.JSONDecodeError: pass # Extract html lang attribute - html_elem = doc.find(".//html") + # ``fromstring`` returns the document's ```` element for a + # complete HTML document. ``.//html`` only searches descendants, so + # it misses that root element and drops its language declaration. + html_elem = doc if doc.tag.lower() == "html" else doc.find(".//html") if html_elem is not None: info["lang"] = html_elem.attrib.get("lang", "") return info diff --git a/crawl4ai/content_scraping_strategy.py b/crawl4ai/content_scraping_strategy.py index 67e87250d..3eb5b6a0b 100644 --- a/crawl4ai/content_scraping_strategy.py +++ b/crawl4ai/content_scraping_strategy.py @@ -84,8 +84,6 @@ def fetch_image_file_size(img, base_url): return None except InvalidSchema: return None - finally: - return class ContentScrapingStrategy(ABC): diff --git a/tests/unit/test_content_scraping_helpers.py b/tests/unit/test_content_scraping_helpers.py new file mode 100644 index 000000000..225fa8e77 --- /dev/null +++ b/tests/unit/test_content_scraping_helpers.py @@ -0,0 +1,11 @@ +from unittest.mock import Mock, patch + +from crawl4ai.content_scraping_strategy import fetch_image_file_size + + +def test_fetch_image_file_size_returns_content_length(): + image = {"src": "/image.png"} + response = Mock(status_code=200, headers={"Content-Length": "1024"}) + + with patch("crawl4ai.content_scraping_strategy.requests.head", return_value=response): + assert fetch_image_file_size(image, "https://example.com/page") == "1024" diff --git a/tests/unit/test_sitemap_namespace_parsing.py b/tests/unit/test_sitemap_namespace_parsing.py index 3370ddb55..64e0cdf68 100644 --- a/tests/unit/test_sitemap_namespace_parsing.py +++ b/tests/unit/test_sitemap_namespace_parsing.py @@ -15,7 +15,15 @@ def get_scores(self, tokens): sys.modules.setdefault("rank_bm25", SimpleNamespace(BM25Okapi=_FakeBM25)) -from crawl4ai.async_url_seeder import AsyncUrlSeeder +from crawl4ai.async_url_seeder import AsyncUrlSeeder, _parse_head + + +def test_parse_head_reads_language_from_document_root(): + info = _parse_head( + 'Başlık' + ) + + assert info["lang"] == "tr" class DummyResponse: