From 8b3a21c10de1269db16292f63d514a65fb1883f6 Mon Sep 17 00:00:00 2001 From: Anas Khan <83116240+anxkhn@users.noreply.github.com> Date: Fri, 3 Jul 2026 10:37:24 +0530 Subject: [PATCH] fix!: normalize_url preserves path, query, and fragment case normalize_url lowercased the entire URL, including the path and query, even though only the scheme and host are case-insensitive per RFC 3986. Because compute_unique_key uses the normalized URL as the default unique_key, any two URLs that differ only in path or query casing (for example /Product/ABC vs /product/abc, or ?token=SeCrEt vs ?token=secret) collided and were silently deduplicated, dropping case-distinct pages with no log or statistic. yarl already lower-cases the scheme and host on parse while preserving the case of the path, query, and fragment, so dropping the trailing whole-string .lower() yields RFC 3986 compliant normalization and matches the behavior of Crawlee for JavaScript. BREAKING CHANGE: default unique keys for URLs with mixed-case paths or queries now differ from previous releases, so such requests are no longer deduplicated together and keys persisted by older versions will not match newly computed ones. Pass an explicit unique_key (or lowercase URLs via transform_request_function before enqueuing) to keep the old behavior. Closes #2008 --- src/crawlee/_utils/requests.py | 11 +++++++---- tests/unit/_utils/test_requests.py | 22 ++++++++++++++++++++-- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/src/crawlee/_utils/requests.py b/src/crawlee/_utils/requests.py index fa31d4621d..87fcb1e436 100644 --- a/src/crawlee/_utils/requests.py +++ b/src/crawlee/_utils/requests.py @@ -17,10 +17,12 @@ def normalize_url(url: str, *, keep_url_fragment: bool = False) -> str: """Normalize a URL. This function cleans and standardizes a URL by removing leading and trailing whitespaces, - converting the scheme and netloc to lower case, stripping unwanted tracking parameters + converting the scheme and host to lower case, stripping unwanted tracking parameters (specifically those beginning with 'utm_'), sorting the remaining query parameters alphabetically, and optionally retaining the URL fragment. The goal is to ensure that URLs that are functionally - identical but differ in trivial ways (such as parameter order or casing) are treated as the same. + identical but differ in trivial ways (such as parameter order or scheme/host casing) are treated + as the same. Per RFC 3986, only the scheme and host are case-insensitive, so the case of the path, + query, and fragment is preserved. Args: url: The URL to be normalized. @@ -38,13 +40,14 @@ def normalize_url(url: str, *, keep_url_fragment: bool = False) -> str: # Construct the new query string sorted_search_params = sorted(search_params) - # Construct the final URL + # Construct the final URL. `yarl` already lower-cases the scheme and host while preserving the + # case of the path, query, and fragment, so no additional lower-casing is applied here. yarl_new_url = parsed_url.with_query(sorted_search_params) yarl_new_url = yarl_new_url.with_path( yarl_new_url.path.removesuffix('/'), keep_query=True, keep_fragment=keep_url_fragment ) - return str(yarl_new_url).lower() + return str(yarl_new_url) def compute_unique_key( diff --git a/tests/unit/_utils/test_requests.py b/tests/unit/_utils/test_requests.py index 8198909592..b4e34fbb33 100644 --- a/tests/unit/_utils/test_requests.py +++ b/tests/unit/_utils/test_requests.py @@ -15,22 +15,26 @@ 'http://example.com/?another_key=another_value&key=value', False, ), - ('HTTPS://EXAMPLE.COM/?KEY=VALUE', 'https://example.com/?key=value', False), + ('HTTPS://EXAMPLE.COM/?KEY=VALUE', 'https://example.com/?KEY=VALUE', False), ('', '', False), ('http://example.com/#fragment', 'http://example.com/#fragment', True), ('http://example.com/#fragment', 'http://example.com', False), (' https://example.com/ ', 'https://example.com', False), ('http://example.com/?b=2&a=1', 'http://example.com/?a=1&b=2', False), + ('https://EXAMPLE.com/Path/To/Page', 'https://example.com/Path/To/Page', False), + ('https://example.com/?token=SeCrEt', 'https://example.com/?token=SeCrEt', False), ], ids=[ 'remove_utm_params', 'retain_sort_non_utm_params', - 'convert_scheme_netloc_to_lowercase', + 'lowercase_scheme_host_only', 'handle_empty_url', 'retain_fragment', 'remove_fragment', 'trim_whitespace', 'sort_query_params', + 'preserve_path_case', + 'preserve_query_case', ], ) def test_normalize_url(url: str, expected_output: str, *, keep_url_fragment: bool) -> None: @@ -38,6 +42,20 @@ def test_normalize_url(url: str, expected_output: str, *, keep_url_fragment: boo assert output == expected_output +def test_normalize_url_preserves_case_distinct_paths_and_queries() -> None: + # URLs that differ only in the case of their path or query must not be collapsed together, + # otherwise `compute_unique_key` would silently deduplicate case-distinct pages (see issue #2008). + assert normalize_url('https://example.com/Product/ABC') != normalize_url('https://example.com/product/abc') + assert normalize_url('https://example.com/?token=SeCrEt') != normalize_url('https://example.com/?token=secret') + + +def test_compute_unique_key_preserves_case_distinct_paths_and_queries() -> None: + assert compute_unique_key('https://example.com/Product/ABC') != compute_unique_key( + 'https://example.com/product/abc' + ) + assert compute_unique_key('https://example.com/?a=SeCrEt') != compute_unique_key('https://example.com/?a=secret') + + def test_compute_unique_key_basic() -> None: url = 'https://crawlee.dev' uk_get = compute_unique_key(url, method='GET')