diff --git a/src/crawlee/_utils/requests.py b/src/crawlee/_utils/requests.py index fa31d4621d..87fcb1e436 100644 --- a/src/crawlee/_utils/requests.py +++ b/src/crawlee/_utils/requests.py @@ -17,10 +17,12 @@ def normalize_url(url: str, *, keep_url_fragment: bool = False) -> str: """Normalize a URL. This function cleans and standardizes a URL by removing leading and trailing whitespaces, - converting the scheme and netloc to lower case, stripping unwanted tracking parameters + converting the scheme and host to lower case, stripping unwanted tracking parameters (specifically those beginning with 'utm_'), sorting the remaining query parameters alphabetically, and optionally retaining the URL fragment. The goal is to ensure that URLs that are functionally - identical but differ in trivial ways (such as parameter order or casing) are treated as the same. + identical but differ in trivial ways (such as parameter order or scheme/host casing) are treated + as the same. Per RFC 3986, only the scheme and host are case-insensitive, so the case of the path, + query, and fragment is preserved. Args: url: The URL to be normalized. @@ -38,13 +40,14 @@ def normalize_url(url: str, *, keep_url_fragment: bool = False) -> str: # Construct the new query string sorted_search_params = sorted(search_params) - # Construct the final URL + # Construct the final URL. `yarl` already lower-cases the scheme and host while preserving the + # case of the path, query, and fragment, so no additional lower-casing is applied here. yarl_new_url = parsed_url.with_query(sorted_search_params) yarl_new_url = yarl_new_url.with_path( yarl_new_url.path.removesuffix('/'), keep_query=True, keep_fragment=keep_url_fragment ) - return str(yarl_new_url).lower() + return str(yarl_new_url) def compute_unique_key( diff --git a/tests/unit/_utils/test_requests.py b/tests/unit/_utils/test_requests.py index 8198909592..b4e34fbb33 100644 --- a/tests/unit/_utils/test_requests.py +++ b/tests/unit/_utils/test_requests.py @@ -15,22 +15,26 @@ 'http://example.com/?another_key=another_value&key=value', False, ), - ('HTTPS://EXAMPLE.COM/?KEY=VALUE', 'https://example.com/?key=value', False), + ('HTTPS://EXAMPLE.COM/?KEY=VALUE', 'https://example.com/?KEY=VALUE', False), ('', '', False), ('http://example.com/#fragment', 'http://example.com/#fragment', True), ('http://example.com/#fragment', 'http://example.com', False), (' https://example.com/ ', 'https://example.com', False), ('http://example.com/?b=2&a=1', 'http://example.com/?a=1&b=2', False), + ('https://EXAMPLE.com/Path/To/Page', 'https://example.com/Path/To/Page', False), + ('https://example.com/?token=SeCrEt', 'https://example.com/?token=SeCrEt', False), ], ids=[ 'remove_utm_params', 'retain_sort_non_utm_params', - 'convert_scheme_netloc_to_lowercase', + 'lowercase_scheme_host_only', 'handle_empty_url', 'retain_fragment', 'remove_fragment', 'trim_whitespace', 'sort_query_params', + 'preserve_path_case', + 'preserve_query_case', ], ) def test_normalize_url(url: str, expected_output: str, *, keep_url_fragment: bool) -> None: @@ -38,6 +42,20 @@ def test_normalize_url(url: str, expected_output: str, *, keep_url_fragment: boo assert output == expected_output +def test_normalize_url_preserves_case_distinct_paths_and_queries() -> None: + # URLs that differ only in the case of their path or query must not be collapsed together, + # otherwise `compute_unique_key` would silently deduplicate case-distinct pages (see issue #2008). + assert normalize_url('https://example.com/Product/ABC') != normalize_url('https://example.com/product/abc') + assert normalize_url('https://example.com/?token=SeCrEt') != normalize_url('https://example.com/?token=secret') + + +def test_compute_unique_key_preserves_case_distinct_paths_and_queries() -> None: + assert compute_unique_key('https://example.com/Product/ABC') != compute_unique_key( + 'https://example.com/product/abc' + ) + assert compute_unique_key('https://example.com/?a=SeCrEt') != compute_unique_key('https://example.com/?a=secret') + + def test_compute_unique_key_basic() -> None: url = 'https://crawlee.dev' uk_get = compute_unique_key(url, method='GET')