fix!: normalize_url preserves path, query, and fragment case

src/crawlee/_utils/requests.py

@@ -17,10 +17,12 @@ def normalize_url(url: str, *, keep_url_fragment: bool = False) -> str:
     """Normalize a URL.
 
     This function cleans and standardizes a URL by removing leading and trailing whitespaces,
-    converting the scheme and netloc to lower case, stripping unwanted tracking parameters
+    converting the scheme and host to lower case, stripping unwanted tracking parameters
     (specifically those beginning with 'utm_'), sorting the remaining query parameters alphabetically,
     and optionally retaining the URL fragment. The goal is to ensure that URLs that are functionally
-    identical but differ in trivial ways (such as parameter order or casing) are treated as the same.
+    identical but differ in trivial ways (such as parameter order or scheme/host casing) are treated
+    as the same. Per RFC 3986, only the scheme and host are case-insensitive, so the case of the path,
+    query, and fragment is preserved.
 
     Args:
         url: The URL to be normalized.
@@ -38,13 +40,14 @@ def normalize_url(url: str, *, keep_url_fragment: bool = False) -> str:
     # Construct the new query string
     sorted_search_params = sorted(search_params)
 
-    # Construct the final URL
+    # Construct the final URL. `yarl` already lower-cases the scheme and host while preserving the
+    # case of the path, query, and fragment, so no additional lower-casing is applied here.
     yarl_new_url = parsed_url.with_query(sorted_search_params)
     yarl_new_url = yarl_new_url.with_path(
         yarl_new_url.path.removesuffix('/'), keep_query=True, keep_fragment=keep_url_fragment
     )
 
-    return str(yarl_new_url).lower()
+    return str(yarl_new_url)
 
 
 def compute_unique_key(

tests/unit/_utils/test_requests.py

@@ -15,29 +15,47 @@
             'http://example.com/?another_key=another_value&key=value',
             False,
         ),
-        ('HTTPS://EXAMPLE.COM/?KEY=VALUE', 'https://example.com/?key=value', False),
+        ('HTTPS://EXAMPLE.COM/?KEY=VALUE', 'https://example.com/?KEY=VALUE', False),
         ('', '', False),
         ('http://example.com/#fragment', 'http://example.com/#fragment', True),
         ('http://example.com/#fragment', 'http://example.com', False),
         ('  https://example.com/  ', 'https://example.com', False),
         ('http://example.com/?b=2&a=1', 'http://example.com/?a=1&b=2', False),
+        ('https://EXAMPLE.com/Path/To/Page', 'https://example.com/Path/To/Page', False),
+        ('https://example.com/?token=SeCrEt', 'https://example.com/?token=SeCrEt', False),
     ],
     ids=[
         'remove_utm_params',
         'retain_sort_non_utm_params',
-        'convert_scheme_netloc_to_lowercase',
+        'lowercase_scheme_host_only',
         'handle_empty_url',
         'retain_fragment',
         'remove_fragment',
         'trim_whitespace',
         'sort_query_params',
+        'preserve_path_case',
+        'preserve_query_case',
     ],
 )
 def test_normalize_url(url: str, expected_output: str, *, keep_url_fragment: bool) -> None:
     output = normalize_url(url, keep_url_fragment=keep_url_fragment)
     assert output == expected_output
 
 
+def test_normalize_url_preserves_case_distinct_paths_and_queries() -> None:
+    # URLs that differ only in the case of their path or query must not be collapsed together,
+    # otherwise `compute_unique_key` would silently deduplicate case-distinct pages (see issue #2008).
+    assert normalize_url('https://example.com/Product/ABC') != normalize_url('https://example.com/product/abc')
+    assert normalize_url('https://example.com/?token=SeCrEt') != normalize_url('https://example.com/?token=secret')
+
+
+def test_compute_unique_key_preserves_case_distinct_paths_and_queries() -> None:
+    assert compute_unique_key('https://example.com/Product/ABC') != compute_unique_key(
+        'https://example.com/product/abc'
+    )
+    assert compute_unique_key('https://example.com/?a=SeCrEt') != compute_unique_key('https://example.com/?a=secret')
+
+
 def test_compute_unique_key_basic() -> None:
     url = 'https://crawlee.dev'
     uk_get = compute_unique_key(url, method='GET')